automated tiers

docs/histograms.txt

@@ -0,0 +1,5 @@
+[2023-06-09T18:45:50Z WARN  web3_proxy::rpcs::consensus] TODO: find the troughs in the histogram: HISTFAAAADV4nC3GoQ0AIAxE0eNAkRAEwbELQxE2QGG7aEeoaL95fz0ZACq8HKbwb/U5bGXystMAZl8EMw==
+
+Paste the HIST data here: <https://hdrhistogram.github.io/HdrHistogramJSDemo/decoding-demo.html>
+
+Save the "Decoded histogram data" to a file and upload it here: <http://hdrhistogram.github.io/HdrHistogram/plotFiles.html>

web3_proxy/Cargo.toml

@@ -100,6 +100,8 @@ tracing-subscriber = "0.3"
 ulid = { version = "1.0.0", features = ["rand", "uuid", "serde"] }
 url = "2.4.0"
 uuid = { version = "1.3.3", default-features = false, features = ["fast-rng", "serde", "v4", "zerocopy"] }
+hdrhistogram = "7.5.2"
+base64 = "0.21.2"
 
 [dev-dependencies]
 tokio = { version = "1.28.2", features = ["full", "test-util"] }

web3_proxy/src/bin/web3_proxy_cli/proxyd.rs

@@ -310,7 +310,6 @@ mod tests {
                     Web3RpcConfig {
                         http_url: Some(anvil.endpoint()),
                         soft_limit: 100,
-                        tier: 0,
                         ..Default::default()
                     },
                 ),
@@ -319,7 +318,6 @@ mod tests {
                     Web3RpcConfig {
                         ws_url: Some(anvil.ws_endpoint()),
                         soft_limit: 100,
-                        tier: 0,
                         ..Default::default()
                     },
                 ),

web3_proxy/src/config.rs

@@ -256,9 +256,6 @@ pub struct Web3RpcConfig {
     /// only use this rpc if everything else is lagging too far. this allows us to ignore fast but very low limit rpcs
     #[serde(default)]
     pub backup: bool,
-    /// All else equal, a server with a lower tier receives all requests
-    #[serde(default = "default_tier")]
-    pub tier: u64,
     /// Subscribe to the firehose of pending transactions
     /// Don't do this with free rpcs
     #[serde(default)]
@@ -268,10 +265,6 @@ pub struct Web3RpcConfig {
     pub extra: HashMap<String, serde_json::Value>,
 }
 
-fn default_tier() -> u64 {
-    0
-}
-
 impl Web3RpcConfig {
     /// Create a Web3Rpc from config
     /// TODO: move this into Web3Rpc? (just need to make things pub(crate))

web3_proxy/src/errors.rs

@@ -61,6 +61,7 @@ pub enum Web3ProxyError {
     EthersWsClient(ethers::prelude::WsClientError),
     FlumeRecv(flume::RecvError),
     GasEstimateNotU256,
+    HdrRecord(hdrhistogram::errors::RecordError),
     Headers(headers::Error),
     HeaderToString(ToStrError),
     Hyper(hyper::Error),
@@ -338,6 +339,17 @@ impl Web3ProxyError {
                     },
                 )
             }
+            Self::HdrRecord(err) => {
+                warn!("HdrRecord {:?}", err);
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    JsonRpcErrorData {
+                        message: format!("{}", err).into(),
+                        code: StatusCode::INTERNAL_SERVER_ERROR.as_u16().into(),
+                        data: None,
+                    },
+                )
+            }
             Self::Headers(err) => {
                 warn!("HeadersError {:?}", err);
                 (

web3_proxy/src/frontend/users/payment.rs

@@ -13,15 +13,13 @@ use ethbloom::Input as BloomInput;
 use ethers::abi::AbiEncode;
 use ethers::types::{Address, TransactionReceipt, ValueOrArray, H256};
 use hashbrown::HashMap;
-// use http::StatusCode;
 use http::StatusCode;
 use log::{debug, info, trace};
 use migration::sea_orm::prelude::Decimal;
 use migration::sea_orm::{
     self, ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait,
 };
 use migration::{Expr, OnConflict};
-use num_traits::Pow;
 use payment_contracts::ierc20::IERC20;
 use payment_contracts::payment_factory::{self, PaymentFactory};
 use serde_json::json;