Merge branch 'master' into feature/dilithium-scheme-integration

.circleci/config.yml

@@ -196,7 +196,7 @@ commands:
           shell: bash.exe
           command: |
             choco install -y msys2 pacman make wget --force
-            choco install -y golang --version=1.16.11 --force
+            choco install -y golang --version=1.14.7 --force
             choco install -y python3 --version=3.7.3 --force
             export msys2='cmd //C RefreshEnv.cmd '
             export msys2+='& set MSYS=winsymlinks:nativestrict '

.github/workflows/reviewdog.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Install specific golang
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16.11'
+          go-version: '1.16.6'
       - name: Create folders for golangci-lint
         run: mkdir -p cicdtmp/golangci-lint
       - name: Check if custom golangci-lint is already built

Makefile

@@ -8,6 +8,7 @@ else
 export GOPATH := $(shell go env GOPATH)
 GOPATH1 := $(firstword $(subst :, ,$(GOPATH)))
 endif
+export GO111MODULE	:= on
 export GOPROXY := direct
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)

agreement/vote.go

@@ -129,7 +129,7 @@ func (uv unauthenticatedVote) verify(l LedgerReader) (vote, error) {
 
 	ephID := basics.OneTimeIDForRound(rv.Round, m.Record.KeyDilution(proto))
 	voteID := m.Record.VoteID
-	if !voteID.Verify(ephID, rv, uv.Sig) {
+	if !voteID.Verify(ephID, rv, uv.Sig, proto.EnableBatchVerification) {
 		return vote{}, fmt.Errorf("unauthenticatedVote.verify: could not verify FS signature on vote by %v given %v: %+v", rv.Sender, voteID, uv)
 	}
 

config/consensus.go

@@ -395,6 +395,9 @@ type ConsensusParams struct {
 	// to be taken offline, that would be proposed to be taken offline.
 	MaxProposedExpiredOnlineAccounts int
 
+	//EnableBatchVerification enable the use of the batch verification algorithm.
+	EnableBatchVerification bool
+
 	// When rewards rate changes, use the new value immediately.
 	RewardsCalculationFix bool
 
@@ -1066,6 +1069,8 @@ func initConsensusProtocols() {
 
 	vFuture.MaxProposedExpiredOnlineAccounts = 32
 
+	vFuture.EnableBatchVerification = true
+
 	vFuture.RewardsCalculationFix = true
 
 	// stat proof key registration

crypto/batchverifier.go

@@ -16,40 +16,78 @@
 
 package crypto
 
-import "errors"
+// #cgo CFLAGS: -Wall -std=c99
+// #cgo darwin,amd64 CFLAGS: -I${SRCDIR}/libs/darwin/amd64/include
+// #cgo darwin,amd64 LDFLAGS: ${SRCDIR}/libs/darwin/amd64/lib/libsodium.a
+// #cgo darwin,arm64 CFLAGS: -I${SRCDIR}/libs/darwin/arm64/include
+// #cgo darwin,arm64 LDFLAGS: ${SRCDIR}/libs/darwin/arm64/lib/libsodium.a
+// #cgo linux,amd64 CFLAGS: -I${SRCDIR}/libs/linux/amd64/include
+// #cgo linux,amd64 LDFLAGS: ${SRCDIR}/libs/linux/amd64/lib/libsodium.a
+// #cgo linux,arm64 CFLAGS: -I${SRCDIR}/libs/linux/arm64/include
+// #cgo linux,arm64 LDFLAGS: ${SRCDIR}/libs/linux/arm64/lib/libsodium.a
+// #cgo linux,arm CFLAGS: -I${SRCDIR}/libs/linux/arm/include
+// #cgo linux,arm LDFLAGS: ${SRCDIR}/libs/linux/arm/lib/libsodium.a
+// #cgo windows,amd64 CFLAGS: -I${SRCDIR}/libs/windows/amd64/include
+// #cgo windows,amd64 LDFLAGS: ${SRCDIR}/libs/windows/amd64/lib/libsodium.a
+// #include <stdint.h>
+// #include "sodium.h"
+// enum {
+//	sizeofPtr = sizeof(void*),
+//	sizeofULongLong = sizeof(unsigned long long),
+// };
+import "C"
+import (
+	"errors"
+	"unsafe"
+)
 
 // BatchVerifier enqueues signatures to be validated in batch.
 type BatchVerifier struct {
-	messages   []Hashable          // contains a slice of messages to be hashed. Each message is varible length
-	publicKeys []SignatureVerifier // contains a slice of public keys. Each individual public key is 32 bytes.
-	signatures []Signature         // contains a slice of signatures keys. Each individual signature is 64 bytes.
+	messages             []Hashable          // contains a slice of messages to be hashed. Each message is varible length
+	publicKeys           []SignatureVerifier // contains a slice of public keys. Each individual public key is 32 bytes.
+	signatures           []Signature         // contains a slice of signatures keys. Each individual signature is 64 bytes.
+	useBatchVerification bool
 }
 
 const minBatchVerifierAlloc = 16
 
 // Batch verifications errors
 var (
 	ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification")
-	ErrZeroTranscationsInBatch = errors.New("Could not validate empty signature set")
+	ErrZeroTransactionInBatch  = errors.New("Could not validate empty signature set")
 )
 
+//export ed25519_randombytes_unsafe
+func ed25519_randombytes_unsafe(p unsafe.Pointer, len C.size_t) {
+	randBuf := (*[1 << 30]byte)(p)[:len:len]
+	RandBytes(randBuf)
+}
+
+// MakeBatchVerifierWithAlgorithmDefaultSize create a BatchVerifier instance. This function pre-allocates
+// amount of free space to enqueue signatures without expanding. this function always use the batch
+// verification algorithm
+func MakeBatchVerifierWithAlgorithmDefaultSize() *BatchVerifier {
+	return MakeBatchVerifier(minBatchVerifierAlloc, true)
+}
+
 // MakeBatchVerifierDefaultSize create a BatchVerifier instance. This function pre-allocates
-// amount of free space to enqueue signatures without exapneding
-func MakeBatchVerifierDefaultSize() *BatchVerifier {
-	return MakeBatchVerifier(minBatchVerifierAlloc)
+// amount of free space to enqueue signatures without expanding
+func MakeBatchVerifierDefaultSize(enableBatchVerification bool) *BatchVerifier {
+	return MakeBatchVerifier(minBatchVerifierAlloc, enableBatchVerification)
 }
 
 // MakeBatchVerifier create a BatchVerifier instance. This function pre-allocates
 // a given space so it will not expaned the storage
-func MakeBatchVerifier(hint int) *BatchVerifier {
+func MakeBatchVerifier(hint int, enableBatchVerification bool) *BatchVerifier {
 	// preallocate enough storage for the expected usage. We will reallocate as needed.
 	if hint < minBatchVerifierAlloc {
 		hint = minBatchVerifierAlloc
 	}
 	return &BatchVerifier{
-		messages:   make([]Hashable, 0, hint),
-		publicKeys: make([]SignatureVerifier, 0, hint),
-		signatures: make([]Signature, 0, hint),
+		messages:             make([]Hashable, 0, hint),
+		publicKeys:           make([]SignatureVerifier, 0, hint),
+		signatures:           make([]Signature, 0, hint),
+		useBatchVerification: enableBatchVerification,
 	}
 }
 
@@ -85,14 +123,69 @@ func (b *BatchVerifier) GetNumberOfEnqueuedSignatures() int {
 // if the batch is zero an appropriate error is return.
 func (b *BatchVerifier) Verify() error {
 	if b.GetNumberOfEnqueuedSignatures() == 0 {
-		return ErrZeroTranscationsInBatch
+		return ErrZeroTransactionInBatch
 	}
 
+	if b.useBatchVerification {
+		var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures())
+		for i, m := range b.messages {
+			messages[i] = HashRep(m)
+		}
+		if batchVerificationImpl(messages, b.publicKeys, b.signatures) {
+			return nil
+		}
+		return ErrBatchVerificationFailed
+	}
+	return b.verifyOneByOne()
+}
+
+func (b *BatchVerifier) verifyOneByOne() error {
 	for i := range b.messages {
-		verifier := SignatureVerifier(b.publicKeys[i])
-		if !verifier.Verify(b.messages[i], b.signatures[i]) {
+		verifier := b.publicKeys[i]
+		if !verifier.Verify(b.messages[i], b.signatures[i], false) {
 			return ErrBatchVerificationFailed
 		}
 	}
 	return nil
 }
+
+// batchVerificationImpl invokes the ed25519 batch verification algorithm.
+// it returns true if all the signatures were authentically signed by the owners
+func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) bool {
+
+	numberOfSignatures := len(messages)
+
+	messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	messagesLenAllocation := C.malloc(C.size_t(C.sizeofULongLong * numberOfSignatures))
+	publicKeysAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	signaturesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	valid := C.malloc(C.size_t(C.sizeof_int * numberOfSignatures))
+
+	defer func() {
+		// release staging memory
+		C.free(messagesAllocation)
+		C.free(messagesLenAllocation)
+		C.free(publicKeysAllocation)
+		C.free(signaturesAllocation)
+		C.free(valid)
+	}()
+
+	// load all the data pointers into the array pointers.
+	for i := 0; i < numberOfSignatures; i++ {
+		*(*uintptr)(unsafe.Pointer(uintptr(messagesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&messages[i][0]))
+		*(*C.ulonglong)(unsafe.Pointer(uintptr(messagesLenAllocation) + uintptr(i*C.sizeofULongLong))) = C.ulonglong(len(messages[i]))
+		*(*uintptr)(unsafe.Pointer(uintptr(publicKeysAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&publicKeys[i][0]))
+		*(*uintptr)(unsafe.Pointer(uintptr(signaturesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&signatures[i][0]))
+	}
+
+	// call the batch verifier
+	allValid := C.crypto_sign_ed25519_open_batch(
+		(**C.uchar)(unsafe.Pointer(messagesAllocation)),
+		(*C.ulonglong)(unsafe.Pointer(messagesLenAllocation)),
+		(**C.uchar)(unsafe.Pointer(publicKeysAllocation)),
+		(**C.uchar)(unsafe.Pointer(signaturesAllocation)),
+		C.size_t(len(messages)),
+		(*C.int)(unsafe.Pointer(valid)))
+
+	return allValid == 0
+}

crypto/batchverifier_test.go

@@ -27,7 +27,7 @@ import (
 func TestBatchVerifierSingle(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	// test expected success
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	msg := randString()
 	var s Seed
 	RandBytes(s[:])
@@ -36,8 +36,8 @@ func TestBatchVerifierSingle(t *testing.T) {
 	bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig)
 	require.NoError(t, bv.Verify())
 
-	// test expected failuire
-	bv = MakeBatchVerifier(1)
+	// test expected failure
+	bv = MakeBatchVerifierWithAlgorithmDefaultSize()
 	msg = randString()
 	RandBytes(s[:])
 	sigSecrets = GenerateSignatureSecrets(s)
@@ -52,7 +52,7 @@ func TestBatchVerifierBulk(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	for i := 1; i < 64*2+3; i++ {
 		n := i
-		bv := MakeBatchVerifier(n)
+		bv := MakeBatchVerifier(n, true)
 		var s Seed
 
 		for i := 0; i < n; i++ {
@@ -71,7 +71,7 @@ func TestBatchVerifierBulk(t *testing.T) {
 func TestBatchVerifierBulkWithExpand(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	n := 64
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	var s Seed
 	RandBytes(s[:])
 
@@ -87,7 +87,7 @@ func TestBatchVerifierBulkWithExpand(t *testing.T) {
 func TestBatchVerifierWithInvalidSiganture(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	n := 64
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	var s Seed
 	RandBytes(s[:])
 
@@ -109,7 +109,7 @@ func TestBatchVerifierWithInvalidSiganture(t *testing.T) {
 
 func BenchmarkBatchVerifier(b *testing.B) {
 	c := makeCurve25519Secret()
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifier(1, true)
 	for i := 0; i < b.N; i++ {
 		str := randString()
 		bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str))
@@ -121,6 +121,6 @@ func BenchmarkBatchVerifier(b *testing.B) {
 
 func TestEmpty(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	bv := MakeBatchVerifierDefaultSize()
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	require.Error(t, bv.Verify())
 }

crypto/compactcert/builder.go

@@ -99,7 +99,6 @@ func (b *Builder) Add(pos uint64, sig merklesignature.Signature, verifySig bool)
 	}
 
 	// Check signature
-
 	if verifySig {
 		if err := p.PK.Verify(uint64(b.SigRound), b.Msg, sig); err != nil {
 			return err

crypto/compactcert/structs.go

@@ -29,6 +29,8 @@ type Params struct {
 	ProvenWeight uint64          // Weight threshold proven by the certificate
 	SigRound     basics.Round    // The round for which the ephemeral key is committed to
 	SecKQ        uint64          // Security parameter (k+q) from analysis document
+
+	EnableBatchVerification bool // whether ED25519 batch verification is enabled
 }
 
 // CompactOneTimeSignature is crypto.OneTimeSignature with omitempty

crypto/crypto_test.go

@@ -43,22 +43,22 @@ func randString() (b TestingHashable) {
 func signVerify(t *testing.T, c *SignatureSecrets, c2 *SignatureSecrets) {
 	s := randString()
 	sig := c.Sign(s)
-	if !c.Verify(s, sig) {
+	if !c.Verify(s, sig, true) {
 		t.Errorf("correct signature failed to verify (plain)")
 	}
 
 	s2 := randString()
 	sig2 := c.Sign(s2)
-	if c.Verify(s, sig2) {
+	if c.Verify(s, sig2, true) {
 		t.Errorf("wrong message incorrectly verified (plain)")
 	}
 
 	sig3 := c2.Sign(s)
-	if c.Verify(s, sig3) {
+	if c.Verify(s, sig3, true) {
 		t.Errorf("wrong key incorrectly verified (plain)")
 	}
 
-	if c.Verify(s2, sig3) {
+	if c.Verify(s2, sig3, true) {
 		t.Errorf("wrong message+key incorrectly verified (plain)")
 	}
 }

crypto/curve25519.go

@@ -112,14 +112,19 @@ func ed25519Sign(secret ed25519PrivateKey, data []byte) (sig ed25519Signature) {
 	return
 }
 
-func ed25519Verify(public ed25519PublicKey, data []byte, sig ed25519Signature) bool {
+func ed25519Verify(public ed25519PublicKey, data []byte, sig ed25519Signature, useBatchVerificationCompatibleVersion bool) bool {
 	// &data[0] will make Go panic if msg is zero length
 	d := (*C.uchar)(C.NULL)
 	if len(data) != 0 {
 		d = (*C.uchar)(&data[0])
 	}
 	// https://download.libsodium.org/doc/public-key_cryptography/public-key_signatures#detached-mode
-	result := C.crypto_sign_ed25519_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	var result C.int
+	if useBatchVerificationCompatibleVersion {
+		result = C.crypto_sign_ed25519_bv_compatible_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	} else {
+		result = C.crypto_sign_ed25519_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	}
 	return result == 0
 }
 
@@ -211,15 +216,15 @@ func (s *SignatureSecrets) SignBytes(message []byte) Signature {
 //
 // It returns true if this is the case; otherwise, it returns false.
 //
-func (v SignatureVerifier) Verify(message Hashable, sig Signature) bool {
+func (v SignatureVerifier) Verify(message Hashable, sig Signature, useBatchVerificationCompatibleVersion bool) bool {
 	cryptoSigSecretsVerifyTotal.Inc(map[string]string{})
-	return ed25519Verify(ed25519PublicKey(v), HashRep(message), ed25519Signature(sig))
+	return ed25519Verify(ed25519PublicKey(v), HashRep(message), ed25519Signature(sig), useBatchVerificationCompatibleVersion)
 }
 
 // VerifyBytes verifies a signature, where the message is not hashed first.
 // Caller is responsible for domain separation.
 // If the message is a Hashable, Verify() can be used instead.
-func (v SignatureVerifier) VerifyBytes(message []byte, sig Signature) bool {
+func (v SignatureVerifier) VerifyBytes(message []byte, sig Signature, useBatchVerificationCompatibleVersion bool) bool {
 	cryptoSigSecretsVerifyBytesTotal.Inc(map[string]string{})
-	return ed25519Verify(ed25519PublicKey(v), message, ed25519Signature(sig))
+	return ed25519Verify(ed25519PublicKey(v), message, ed25519Signature(sig), useBatchVerificationCompatibleVersion)
 }