vm subprovider with state hooks

promentol · Dec 14, 2015 · 0256127 · 0256127
1 parent 351b541
commit 0256127
Show file tree

Hide file tree

Showing 7 changed files with 285 additions and 122 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,11 @@
-- web3
-- engine
-- sources
+### 'zero-client' web3 provider
 
+Here’s an explanation of what I’m currently working on:
 
-remote-data adds caching
+a 'zero-client' web3 provider — it is very modular and works via a stack of ‘sub-providers’ which are like normal web3 providers but only handle a subset of rpc methods, specified via `subProvider.methods = [‘eth_call’, ’etc...']`. The intention is to handle as many requests locally as possible, and just let data lookups fallback to some data source ( hosted rpc, blockapps, etc ). Categorically, we don’t want / can’t have the following types of RPC calls go to the network:
+* id mgmt + tx signing (requires private data)
+* filters (requires a stateful data api)
+* vm (expensive, hard to scale)
+The subproviders can emit new rpc requests in order to handle their own;  e.g. `eth_call` may trigger `eth_getAccountBalance`, `eth_getCode`, and others.
+
+'zero-client' web3 provider also handles caching of rpc requests
diff --git a/index.js b/index.js
@@ -18,55 +18,61 @@ var vmSubprovider = new VmSubprovider({
 engine.addSource(vmSubprovider)
 
 
-
-
-// var provider = new Skeleton({
-//   syncProviders: [
-//     keyManager,
-//   ],
-//   asyncProviders: [
-//     keyManager,
-//     blockappsProvider,
-//   ],
-// })
-
-// http://etherscan.io/address/0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae
-vmSubprovider._createAccountStorageTrie('0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae', function(err, storage){
-
-  setInterval(function(){
-
-    storage.get('0x0000000000000000000000000000000000000000000000000000000000000000', function(err, result){
-      console.log('storage result:', result.toString('hex'))
-    })
-
-  }, 1000)
-
+rpcSubprovider.on('block', function(block){
+  // lazy hack - move caching and current block to engine
+  engine.currentBlock = block
+  console.log('================================')
+  console.log('BLOCK CHANGED:', '#'+block.number.toString('hex'), '0x'+block.hash.toString('hex'))
+  console.log('================================')
 })
 
 setInterval(function(){
 
-  // vmSubprovider._fetchAccount('ba0bab3b85c6d190af0f9dbf21a1d8cdbed23830', function(err, account){
-  //   console.log('fetched:', account)
-  // })
+  vmSubprovider.handleAsync({
+    method: 'eth_estimateGas',
+    params: [{
+      from: '0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae',
+      to: '0xba0bab3b85c6d190af0f9dbf21a1d8cdbed23830',
+      value: '0x1234',
+    }, 'latest']
+  }, function(err, results){
+    if (err) throw err
+    console.log('vm results:', results)
+  })
+
+}, 800)
+
+// setInterval(function(){
+
+//   vmSubprovider._fetchAccount('ba0bab3b85c6d190af0f9dbf21a1d8cdbed23830', function(err, account){
+//     console.log('fetched:', account)
+//   })
+
+//   web3.eth.getBalance('ba0bab3b85c6d190af0f9dbf21a1d8cdbed23830', function(err, result){
+//     if (!err)
+//       console.log('looked up baobab bal:', web3.fromWei(result, 'ether').toString())
+//     else
+//       console.error(err);
+//   })
+
+//   web3.eth.getBlock('latest', function(err, result){
+//       if (!err)
+//         console.log('looked up block num:', result.number)
+//       else
+//         console.error(err);
+//   })
+
+// }, 1000)
+
+// http://etherscan.io/address/0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae
+// vmSubprovider._createAccountStorageTrie('0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae', function(err, storage){
 
-  // web3.eth.getBalance('ba0bab3b85c6d190af0f9dbf21a1d8cdbed23830', function(err, result){
-  //   if (!err)
-  //     console.log('looked up baobab bal:', web3.fromWei(result, 'ether').toString())
-  //   else
-  //     console.error(err);
-  // })
+//   setInterval(function(){
 
-  // web3.eth.getBlock('latest', function(err, result){
-  //     if (!err)
-  //       console.log('looked up block num:', result.number)
-  //     else
-  //       console.error(err);
-  // })
+//     storage.get('0x0000000000000000000000000000000000000000000000000000000000000000', function(err, result){
+//       console.log('storage result:', result.toString('hex'))
+//     })
 
-}, 1000)
+//   }, 1000)
 
-rpcSubprovider.on('block', function(block){
-  console.log('================================')
-  console.log('BLOCK CHANGED:', '#'+block.number.toString(), '0x'+block.hash.toString('hex'))
-  console.log('================================')
-})
+// })
diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
     "array.prototype.find": "^1.0.0",
     "async": "^1.5.0",
     "es6-promise": "^3.0.2",
+    "ethereumjs-block": "^1.0.4",
     "ethereumjs-tx": "^0.6.11",
     "ethereumjs-util": "^2.3.1",
     "ethereumjs-vm": "^1.0.3",

diff --git a/subproviders/remote-data.js b/subproviders/remote-data.js
@@ -51,16 +51,24 @@ function RemoteDataSource(){
   ]
 }
 
+RemoteDataSource.prototype.cacheForBlockTag = function(blockTag){
+  const self = this
+  var cache = self._blockCache[blockTag]
+  // if (!cache) cache = self._blockCache[blockTag] = {}
+  return cache
+}
+
 RemoteDataSource.prototype.handleAsync = function(payload, cb){
   const self = this
   self._ready.await(function(){
 
     // parse blockTag
+    // console.log('remote-data:', payload)
     var blockTag = blockTagForPayload(payload)
     if (blockTag === 'latest') blockTag = formatBlockNumber(self._blocks.latest)
 
     // check cache
-    var blockCache = self._blockCache[blockTag]
+    var blockCache = self.cacheForBlockTag(blockTag)
     var requestIdentifier = requestIdentifierForPayload(payload)
     if (blockCache) {
       var result = blockCache[requestIdentifier]
@@ -79,6 +87,7 @@ RemoteDataSource.prototype.handleAsync = function(payload, cb){
         blockCache[requestIdentifier] = result
         console.log('CACHE POPULATE:', blockTag, requestIdentifier, '->', result)
       } else {
+        console.log(self._blockCache)
         console.log('CACHE POPULATE MISS:', blockTag, requestIdentifier)
       }
       cb(null, result)
@@ -97,9 +106,24 @@ RemoteDataSource.prototype.fetchBlock = function(number, cb){
     if (err) return cb(err)
 
     var block = {
-      hash: new Buffer(ethUtil.stripHexPrefix(data.hash), 'hex'),
-      number: new BN(ethUtil.stripHexPrefix(data.number), 16),
-      stateRoot: new Buffer(ethUtil.stripHexPrefix(data.stateRoot), 16),
+      // number: new BN(ethUtil.stripHexPrefix(data.number), 16),
+      number: hexToBuffer(data.number),
+      hash: hexToBuffer(data.hash),
+      parentHash: hexToBuffer(data.parentHash),
+      nonce: hexToBuffer(data.nonce),
+      sha3Uncles: hexToBuffer(data.sha3Uncles),
+      logsBloom: hexToBuffer(data.logsBloom),
+      transactionsRoot: hexToBuffer(data.transactionsRoot),
+      stateRoot: hexToBuffer(data.stateRoot),
+      receiptRoot: hexToBuffer(data.receiptRoot),
+      miner: hexToBuffer(data.miner),
+      difficulty: hexToBuffer(data.difficulty),
+      totalDifficulty: hexToBuffer(data.totalDifficulty),
+      size: hexToBuffer(data.size),
+      extraData: hexToBuffer(data.extraData),
+      gasLimit: hexToBuffer(data.gasLimit),
+      gasUsed: hexToBuffer(data.gasUsed),
+      timestamp: hexToBuffer(data.timestamp),
       transactions: data.transactions,
     }
 
@@ -135,6 +159,7 @@ RemoteDataSource.prototype.setCurrentBlock = function(block){
   var blockNumber = formatBlockNumber(block)
   self._blocks[blockNumber] = block
   self._blocks.latest = block
+  console.log('saving block cache with number:', blockNumber)
   self._blockCache[blockNumber] = {}
   self.emit('block', block)
 }
@@ -145,8 +170,8 @@ RemoteDataSource.prototype.setCurrentBlock = function(block){
 // util
 
 function formatBlockNumber(block){
-  // return ethUtil.addHexPrefix(block.number.toString())
-  return block.number.toString()
+  return ethUtil.addHexPrefix(block.number.toString('hex'))
+  // return block.number.toString()
 }
 
 function requestIdentifierForPayload(payload){
@@ -183,4 +208,10 @@ function paramsWithoutBlockTag(payload){
     default:
       return payload.params.slice()
   }
+}
+
+function hexToBuffer(hexString){
+  hexString = ethUtil.stripHexPrefix(hexString)
+  if (hexString.length%2) hexString = '0'+hexString
+  return new Buffer(hexString, 'hex')
 }
diff --git a/subproviders/rpc-source.js b/subproviders/rpc-source.js
@@ -87,6 +87,7 @@ RpcSource.prototype.requestFromRpc = function(method, params, cb){
     rejectUnauthorized: false,
   }, function(err, res, body) {
     if (err) return cb(err)
+
     // parse response into raw account
     var data
     try {
@@ -100,6 +101,8 @@ RpcSource.prototype.requestFromRpc = function(method, params, cb){
     // console.log(payload, '->', data)
     // console.log('---------------------------------------------')
 
+    if (data.error) return cb(new Error(data.error.message))  
+
     cb(null, data.result)
   })