Further fixes to block test wrapper

* Move go test wrapper for block tests from cmd/geth to tests
* Fix logic for when tests are valid or not, by adding correct
  validations for expected valid/invalid blocks
* Change block insertion helper to work on single blocks
* Add one test case for each file in BlockTests and comment out
  the tests which are currently failing
* Add Skip call in all block tests in lieu of performance fixes
  around ethash cache which are needed before it will be fast enough
  to start / stop the node between each test

cmd/geth/blocktest.go

@@ -109,10 +109,10 @@ func runOneBlockTest(ctx *cli.Context, test *tests.BlockTest) (*eth.Ethereum, er
 		return ethereum, fmt.Errorf("InsertPreState: %v", err)
 	}
 
-	// insert the test blocks, which will execute all transactions
-	if err := test.InsertBlocks(ethereum.ChainManager()); err != nil {
-		return ethereum, fmt.Errorf("Block Test load error: %v %T", err, err)
+	if err := test.TryBlocksInsert(ethereum.ChainManager()); err != nil {
+		return ethereum, fmt.Errorf("Block Test load error: %v", err)
 	}
+
 	fmt.Println("chain loaded")
 	if err := test.ValidatePostState(statedb); err != nil {
 		return ethereum, fmt.Errorf("post state validation failed: %v", err)

tests/block_test.go

@@ -0,0 +1,126 @@
+package tests
+
+import (
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// TODO: refactor test setup & execution to better align with vm and tx tests
+func TestBcValidBlockTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcValidBlockTest.json", []string{}, t)
+}
+
+func TestBcUncleTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcUncleTest.json", []string{}, t)
+}
+
+func TestBcUncleHeaderValidityTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcUncleHeaderValiditiy.json", []string{}, t)
+}
+
+func TestBcInvalidHeaderTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	snafus := []string{
+		"wrongUncleHash", // TODO: why does this fail?
+	}
+	runBlockTestsInFile("files/BlockTests/bcInvalidHeaderTest.json", snafus, t)
+}
+
+func TestBcInvalidRLPTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	snafus := []string{
+		// TODO: why does these fail?
+		"TRANSCT__ZeroByteAtTheEnd",
+		"TRANSCT__RandomByteAtTheEnd",
+		"BLOCK__ZeroByteAtTheEnd",
+		"BLOCK__RandomByteAtTheEnd",
+	}
+	runBlockTestsInFile("files/BlockTests/bcInvalidRLPTest.json", snafus, t)
+}
+
+func TestBcJSAPITests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcJS_API_Test.json", []string{}, t)
+}
+
+func TestBcRPCAPITests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcRPC_API_Test.json", []string{}, t)
+}
+
+func TestBcForkBlockTests(t *testing.T) {
+	t.Skip("Skipped in lieu of performance fixes.")
+	runBlockTestsInFile("files/BlockTests/bcForkBlockTest.json", []string{}, t)
+}
+
+func runBlockTestsInFile(filepath string, snafus []string, t *testing.T) {
+	bt, err := LoadBlockTests(filepath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	notWorking := make(map[string]bool, 100)
+	for _, name := range snafus {
+		notWorking[name] = true
+	}
+
+	for name, test := range bt {
+		if !notWorking[name] {
+			runBlockTest(name, test, t)
+		}
+	}
+}
+
+func runBlockTest(name string, test *BlockTest, t *testing.T) {
+	t.Log("Running test: ", name)
+	cfg := testEthConfig()
+	ethereum, err := eth.New(cfg)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	err = ethereum.Start()
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	// import the genesis block
+	ethereum.ResetWithGenesisBlock(test.Genesis)
+
+	// import pre accounts
+	statedb, err := test.InsertPreState(ethereum.StateDb())
+	if err != nil {
+		t.Fatalf("InsertPreState: %v", err)
+	}
+
+	err = test.TryBlocksInsert(ethereum.ChainManager())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = test.ValidatePostState(statedb); err != nil {
+		t.Fatal("post state validation failed: %v", err)
+	}
+	t.Log("Test passed:  ", name)
+}
+
+func testEthConfig() *eth.Config {
+	ks := crypto.NewKeyStorePassphrase(path.Join(common.DefaultDataDir(), "keys"))
+
+	return &eth.Config{
+		DataDir:        common.DefaultDataDir(),
+		LogLevel:       5,
+		Etherbase:      "primary",
+		AccountManager: accounts.NewManager(ks),
+		NewDB:          func(path string) (common.Database, error) { return ethdb.NewMemDatabase() },
+	}
+}

tests/block_test_util.go

@@ -19,13 +19,27 @@ import (
 )
 
 // Block Test JSON Format
+type BlockTest struct {
+	Genesis *types.Block
+
+	Json        *btJSON
+	preAccounts map[string]btAccount
+}
+
 type btJSON struct {
 	Blocks             []btBlock
 	GenesisBlockHeader btHeader
 	Pre                map[string]btAccount
 	PostState          map[string]btAccount
 }
 
+type btBlock struct {
+	BlockHeader  *btHeader
+	Rlp          string
+	Transactions []btTransaction
+	UncleHeaders []*btHeader
+}
+
 type btAccount struct {
 	Balance string
 	Code    string
@@ -65,20 +79,6 @@ type btTransaction struct {
 	Value    string
 }
 
-type btBlock struct {
-	BlockHeader  *btHeader
-	Rlp          string
-	Transactions []btTransaction
-	UncleHeaders []*btHeader
-}
-
-type BlockTest struct {
-	Genesis *types.Block
-
-	json        *btJSON
-	preAccounts map[string]btAccount
-}
-
 // LoadBlockTests loads a block test JSON file.
 func LoadBlockTests(file string) (map[string]*BlockTest, error) {
 	bt := make(map[string]*btJSON)
@@ -125,13 +125,43 @@ func (t *BlockTest) InsertPreState(db common.Database) (*state.StateDB, error) {
 	return statedb, nil
 }
 
-// InsertBlocks loads the test's blocks into the given chain.
-func (t *BlockTest) InsertBlocks(chain *core.ChainManager) error {
-	blocks, err := t.convertBlocks()
-	if err != nil {
-		return err
+/* See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II
+
+   Whether a block is valid or not is a bit subtle, it's defined by presence of
+   blockHeader, transactions and uncleHeaders fields. If they are missing, the block is
+   invalid and we must verify that we do not accept it.
+
+   Since some tests mix valid and invalid blocks we need to check this for every block.
+
+   If a block is invalid it does not necessarily fail the test, if it's invalidness is
+   expected we are expected to ignore it and continue processing and then validate the
+   post state.
+*/
+func (t *BlockTest) TryBlocksInsert(chainManager *core.ChainManager) error {
+	// insert the test blocks, which will execute all transactions
+	for _, b := range t.Json.Blocks {
+		cb, err := mustConvertBlock(b)
+		if err != nil {
+			if b.BlockHeader == nil {
+				continue // OK - block is supposed to be invalid, continue with next block
+			} else {
+				return fmt.Errorf("Block RLP decoding failed when expected to succeed: ", err)
+			}
+		}
+		// RLP decoding worked, try to insert into chain:
+		err = chainManager.InsertChain(types.Blocks{cb})
+		if err != nil {
+			if b.BlockHeader == nil {
+				continue // OK - block is supposed to be invalid, continue with next block
+			} else {
+				return fmt.Errorf("Block insertion into chain failed: ", err)
+			}
+		}
+		if b.BlockHeader == nil {
+			return fmt.Errorf("Block insertion should have failed")
+		}
 	}
-	return chain.InsertChain(blocks)
+	return nil
 }
 
 func (t *BlockTest) ValidatePostState(statedb *state.StateDB) error {
@@ -159,21 +189,6 @@ func (t *BlockTest) ValidatePostState(statedb *state.StateDB) error {
 	return nil
 }
 
-func (t *BlockTest) convertBlocks() (blocks []*types.Block, err error) {
-	// the conversion handles errors by catching panics.
-	// you might consider this ugly, but the alternative (passing errors)
-	// would be much harder to read.
-	defer func() {
-		if recovered := recover(); recovered != nil {
-			buf := make([]byte, 64<<10)
-			buf = buf[:runtime.Stack(buf, false)]
-			err = fmt.Errorf("%v\n%s", recovered, buf)
-		}
-	}()
-	blocks = mustConvertBlocks(t.json.Blocks)
-	return blocks, nil
-}
-
 func convertTest(in *btJSON) (out *BlockTest, err error) {
 	// the conversion handles errors by catching panics.
 	// you might consider this ugly, but the alternative (passing errors)
@@ -185,7 +200,7 @@ func convertTest(in *btJSON) (out *BlockTest, err error) {
 			err = fmt.Errorf("%v\n%s", recovered, buf)
 		}
 	}()
-	out = &BlockTest{preAccounts: in.Pre, json: in}
+	out = &BlockTest{preAccounts: in.Pre, Json: in}
 	out.Genesis = mustConvertGenesis(in.GenesisBlockHeader)
 	return out, err
 }
@@ -221,17 +236,11 @@ func mustConvertHeader(in btHeader) *types.Header {
 	return header
 }
 
-func mustConvertBlocks(testBlocks []btBlock) []*types.Block {
-	var out []*types.Block
-	for i, inb := range testBlocks {
-		var b types.Block
-		r := bytes.NewReader(mustConvertBytes(inb.Rlp))
-		if err := rlp.Decode(r, &b); err != nil {
-			panic(fmt.Errorf("invalid block %d: %q\nerror: %v", i, inb.Rlp, err))
-		}
-		out = append(out, &b)
-	}
-	return out
+func mustConvertBlock(testBlock btBlock) (*types.Block, error) {
+	var b types.Block
+	r := bytes.NewReader(mustConvertBytes(testBlock.Rlp))
+	err := rlp.Decode(r, &b)
+	return &b, err
 }
 
 func mustConvertBytes(in string) []byte {