[state-sync] add timestamp gauges

this helps us look at how far nodes are behind from a different perspective

Closes: aptos-labs#6774

state-synchronizer/src/coordinator.rs

@@ -461,6 +461,19 @@ impl<T: ExecutorProxyTrait> SyncCoordinator<T> {
         let local_epoch = self.local_state.epoch();
         counters::set_version(counters::VersionType::Synced, synced_version);
         counters::set_version(counters::VersionType::Committed, committed_version);
+        counters::set_timestamp(
+            counters::TimestampType::Synced,
+            self.executor_proxy.get_version_timestamp(synced_version)?,
+        );
+        counters::set_timestamp(
+            counters::TimestampType::Committed,
+            self.executor_proxy
+                .get_version_timestamp(committed_version)?,
+        );
+        counters::set_timestamp(
+            counters::TimestampType::Real,
+            libra_infallible::duration_since_epoch().as_micros() as u64,
+        );
         counters::EPOCH.set(local_epoch as i64);
         debug!(LogSchema::new(LogEntry::LocalState)
             .local_li_version(committed_version)

state-synchronizer/src/counters.rs

@@ -18,7 +18,31 @@ pub const COMMIT_MSG_LABEL: &str = "commit";
 pub const CHUNK_REQUEST_MSG_LABEL: &str = "chunk_request";
 pub const CHUNK_RESPONSE_MSG_LABEL: &str = "chunk_response";
 
-// version type labels
+pub fn set_timestamp(timestamp_type: TimestampType, time_as_usecs: u64) {
+    TIMESTAMP
+        .with_label_values(&[timestamp_type.as_str()])
+        .set((time_as_usecs / 1000) as i64)
+}
+
+pub enum TimestampType {
+    /// Current ledger committed timestamp
+    Committed,
+    /// Current computers clock
+    Real,
+    /// Current ledger synced timestamp
+    Synced,
+}
+
+impl TimestampType {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            TimestampType::Committed => "committed",
+            TimestampType::Real => "real",
+            TimestampType::Synced => "synced",
+        }
+    }
+}
+
 pub fn set_version(version_type: VersionType, version: u64) {
     VERSION
         .with_label_values(&[version_type.as_str()])
@@ -158,6 +182,15 @@ pub static MULTICAST_LEVEL: Lazy<IntGauge> = Lazy::new(|| {
     .unwrap()
 });
 
+pub static TIMESTAMP: Lazy<IntGaugeVec> = Lazy::new(|| {
+    register_int_gauge_vec!(
+        "libra_state_sync_timestamp",
+        "Timestamp involved in state sync progress",
+        &["type"] // see TimestampType above
+    )
+    .unwrap()
+});
+
 /// Notice: this metric is used in CT full node health check
 /// ~/libra/testsuite/cluster-test/health/fullnode_check.rs
 /// please make corresponding changes if this field is updated

state-synchronizer/src/executor_proxy.rs

@@ -49,6 +49,9 @@ pub trait ExecutorProxyTrait: Send {
     /// Get ledger info at an epoch boundary version.
     fn get_epoch_ending_ledger_info(&self, version: u64) -> Result<LedgerInfoWithSignatures>;
 
+    /// Returns the ledger's timestamp for the given version in microseconds
+    fn get_version_timestamp(&self, version: u64) -> Result<u64>;
+
     /// Load all on-chain configs from storage
     /// Note: this method is being exposed as executor proxy trait temporarily because storage read is currently
     /// using the tonic storage read client, which needs the tokio runtime to block on with no runtime/async issues
@@ -195,6 +198,10 @@ impl ExecutorProxyTrait for ExecutorProxy {
         self.storage.get_epoch_ending_ledger_info(version)
     }
 
+    fn get_version_timestamp(&self, version: u64) -> Result<u64> {
+        self.storage.get_block_timestamp(version)
+    }
+
     fn load_on_chain_configs(&mut self) -> Result<()> {
         self.on_chain_configs = Self::fetch_all_configs(&*self.storage)?;
         Ok(())

state-synchronizer/src/tests/helpers.rs

@@ -152,6 +152,11 @@ impl ExecutorProxyTrait for MockExecutorProxy {
         self.storage.read().get_epoch_ending_ledger_info(version)
     }
 
+    fn get_version_timestamp(&self, _version: u64) -> Result<u64> {
+        // Only used for logging purposes so no point in mocking
+        Ok(0)
+    }
+
     fn load_on_chain_configs(&mut self) -> Result<()> {
         Ok(())
     }