From 74626d53d1ab0a82365776d29d9fb03b07335599 Mon Sep 17 00:00:00 2001 From: Joshua Lind Date: Wed, 27 Oct 2021 16:12:20 -0700 Subject: [PATCH] [State Sync] Update Data Streaming Client API to support stream termination. --- state-sync/diem-data-client/src/lib.rs | 8 +- .../src/data_notification.rs | 8 -- .../data-streaming-service/src/data_stream.rs | 49 ++++++---- .../data-streaming-service/src/logging.rs | 1 + .../src/stream_progress_tracker.rs | 2 +- .../src/streaming_client.rs | 77 +++++++++------- .../src/streaming_service.rs | 91 ++++++++++++++++++- .../src/tests/streaming_client.rs | 27 +++--- .../src/tests/streaming_service.rs | 79 ++++++++++++++-- .../data-streaming-service/src/tests/utils.rs | 2 +- 10 files changed, 255 insertions(+), 89 deletions(-) diff --git a/state-sync/diem-data-client/src/lib.rs b/state-sync/diem-data-client/src/lib.rs index 00d3a943d2..7a9d639d33 100644 --- a/state-sync/diem-data-client/src/lib.rs +++ b/state-sync/diem-data-client/src/lib.rs @@ -17,6 +17,8 @@ use storage_service::UnexpectedResponseError; use storage_service_types::{self as storage_service, CompleteDataRange, Epoch}; use thiserror::Error; +pub type ResponseId = u64; + pub mod diemnet; pub type Result = ::std::result::Result; @@ -55,8 +57,8 @@ impl From for Error { /// the Data Client about invalid or malformed responses. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum ResponseError { + InvalidData, InvalidPayloadDataType, - MissingData, ProofVerificationError, } @@ -139,7 +141,7 @@ pub trait DiemDataClient { /// `notify_bad_response()` API call above. #[derive(Clone, Debug)] pub struct Response { - pub id: u64, + pub id: ResponseId, pub payload: T, } @@ -152,7 +154,7 @@ impl Response { self.payload } - pub fn into_parts(self) -> (u64, T) { + pub fn into_parts(self) -> (ResponseId, T) { (self.id, self.payload) } diff --git a/state-sync/state-sync-v2/data-streaming-service/src/data_notification.rs b/state-sync/state-sync-v2/data-streaming-service/src/data_notification.rs index 242e949c4b..f3f7cc352a 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/data_notification.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/data_notification.rs @@ -39,14 +39,6 @@ pub enum DataPayload { TransactionsWithProof(TransactionListWithProof), } -/// A sent data notification that tracks the original client request and the -/// client response forwarded along a stream. This is useful for re-fetching. -#[derive(Clone, Debug)] -pub struct SentDataNotification { - pub client_request: DataClientRequest, - pub client_response: DataClientResponse, -} - /// A request that has been sent to the Diem data client. #[derive(Clone, Debug, Eq, PartialEq)] pub enum DataClientRequest { diff --git a/state-sync/state-sync-v2/data-streaming-service/src/data_stream.rs b/state-sync/state-sync-v2/data-streaming-service/src/data_stream.rs index efc87aa561..23917afc64 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/data_stream.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/data_stream.rs @@ -5,7 +5,6 @@ use crate::{ data_notification, data_notification::{ DataClientRequest, DataClientResponse, DataNotification, DataPayload, NotificationId, - SentDataNotification, }, error::Error, logging::{LogEntry, LogEvent, LogSchema}, @@ -14,7 +13,7 @@ use crate::{ }; use channel::{diem_channel, message_queues::QueueStyle}; use diem_data_client::{ - AdvertisedData, DiemDataClient, GlobalDataSummary, ResponseError, ResponsePayload, + AdvertisedData, DiemDataClient, GlobalDataSummary, ResponseError, ResponseId, ResponsePayload, }; use diem_id_generator::{IdGenerator, U64IdGenerator}; use diem_infallible::Mutex; @@ -65,8 +64,10 @@ pub struct DataStream { // a data notification can be created and sent along the stream. sent_data_requests: Option>, - // The data notifications already sent via this stream. - sent_notifications: HashMap, + // TODO(joshlind): garbage collect me! + // Maps a notification ID (sent along the data stream) to a response ID + // received from the data client. This is useful for providing feedback. + notifications_to_responses: HashMap, // The channel on which to send data notifications when they are ready. notification_sender: channel::diem_channel::Sender<(), DataNotification>, @@ -100,7 +101,7 @@ impl DataStream { diem_data_client, stream_progress_tracker, sent_data_requests: None, - sent_notifications: HashMap::new(), + notifications_to_responses: HashMap::new(), notification_sender, notification_id_generator, stream_end_notification_id: None, @@ -126,6 +127,18 @@ impl DataStream { self.create_and_send_client_requests(&global_data_summary) } + /// Returns the response ID associated with a given `notification_id`. + /// Note: this method returns `None` if the `notification_id` does not match + /// a notification sent via this stream. + pub fn get_response_id_for_notification( + &self, + notification_id: &NotificationId, + ) -> Option { + self.notifications_to_responses + .get(notification_id) + .cloned() + } + /// Creates and sends a batch of diem data client requests to the network fn create_and_send_client_requests( &mut self, @@ -406,18 +419,15 @@ impl DataStream { self.notification_id_generator.clone(), )? { - // Create and save the data notification to track any future re-fetches - let sent_data_notification = SentDataNotification { - client_request: data_client_request.clone(), - client_response: data_client_response.clone(), - }; - if let Some(existing_notification) = self - .sent_notifications - .insert(data_notification.notification_id, sent_data_notification) + // Save the data notification ID and response ID + let notification_id = data_notification.notification_id; + if let Some(response_id) = self + .notifications_to_responses + .insert(notification_id, data_client_response.id) { panic!( - "Duplicate sent notification found! This should not occur! ID: {}, notification: {:?}", - data_notification.notification_id, existing_notification + "Duplicate sent notification ID found! Notification ID: {:?}, Response ID: {:?}", + notification_id, response_id, ); } @@ -426,7 +436,10 @@ impl DataStream { (LogSchema::new(LogEntry::StreamNotification) .stream_id(self.data_stream_id) .event(LogEvent::Success) - .message("Sent a single stream notification!")) + .message(&format!( + "Sent a single stream notification! Notification ID: {:?}", + notification_id + ))) ); self.send_data_notification(data_notification)?; } @@ -463,10 +476,10 @@ impl DataStream { &mut self, ) -> ( &mut Option>, - &mut HashMap, + &mut HashMap, ) { let sent_requests = &mut self.sent_data_requests; - let sent_notifications = &mut self.sent_notifications; + let sent_notifications = &mut self.notifications_to_responses; (sent_requests, sent_notifications) } diff --git a/state-sync/state-sync-v2/data-streaming-service/src/logging.rs b/state-sync/state-sync-v2/data-streaming-service/src/logging.rs index abf840493a..79ed31a9c1 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/logging.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/logging.rs @@ -32,6 +32,7 @@ pub enum LogEntry { CheckStreamProgress, DiemDataClient, EndOfStreamNotification, + HandleTerminateRequest, HandleStreamRequest, InitializeStream, ReceivedDataResponse, diff --git a/state-sync/state-sync-v2/data-streaming-service/src/stream_progress_tracker.rs b/state-sync/state-sync-v2/data-streaming-service/src/stream_progress_tracker.rs index 6dc4eeb7e8..d3af2fe905 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/stream_progress_tracker.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/stream_progress_tracker.rs @@ -119,7 +119,7 @@ impl StreamProgressTracker { Ok(TransactionStreamTracker::new(stream_request)?.into()) } _ => Err(Error::UnsupportedRequestEncountered(format!( - "Stream request not currently supported: {:?}", + "Stream request not supported: {:?}", stream_request ))), } diff --git a/state-sync/state-sync-v2/data-streaming-service/src/streaming_client.rs b/state-sync/state-sync-v2/data-streaming-service/src/streaming_client.rs index 05a2602f6f..d1c87fbc5e 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/streaming_client.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/streaming_client.rs @@ -59,19 +59,21 @@ pub trait DataStreamingClient { max_proof_version: Version, ) -> Result; - /// Refetches the payload for the data notification corresponding to the - /// specified `notification_id`. + /// Terminates the stream that sent the notification with the given + /// `notification_id` and provides feedback for why the payload in the + /// notification was invalid. /// - /// Note: this is required because data payloads may be invalid, e.g., due - /// to invalid or malformed data returned by a misbehaving peer or a failure - /// to verify a proof. The refetch request forces a refetch of the payload - /// and the `refetch_reason` notifies the streaming service as to why the - /// payload must be refetched. - async fn refetch_notification_payload( + /// Note: + /// 1. This is required because data payloads may be invalid, e.g., due to + /// malformed data returned by a misbehaving peer or an invalid proof. This + /// notifies the streaming service that the payload was invalid and thus the + /// stream should be terminated and the responsible peer penalized. + /// 2. Clients that wish to continue fetching data need to open a new stream. + async fn terminate_stream_with_feedback( &self, notification_id: NotificationId, - refetch_reason: PayloadRefetchReason, - ) -> Result; + payload_feedback: PayloadFeedback, + ) -> Result<(), Error>; /// Continuously streams transactions with proofs as the blockchain grows. /// The stream starts at `start_version` and `start_epoch` (inclusive). @@ -114,7 +116,7 @@ pub enum StreamRequest { GetAllTransactionOutputs(GetAllTransactionOutputsRequest), ContinuouslyStreamTransactions(ContinuouslyStreamTransactionsRequest), ContinuouslyStreamTransactionOutputs(ContinuouslyStreamTransactionOutputsRequest), - RefetchNotificationPayload(RefetchNotificationPayloadRequest), + TerminateStream(TerminateStreamRequest), } /// A client request for fetching all account states at a specified version. @@ -161,16 +163,16 @@ pub struct ContinuouslyStreamTransactionOutputsRequest { pub start_epoch: Epoch, } -/// A client request for refetching a notification payload. +/// A client request for terminating a stream and providing payload feedback. #[derive(Clone, Debug, Eq, PartialEq)] -pub struct RefetchNotificationPayloadRequest { +pub struct TerminateStreamRequest { pub notification_id: NotificationId, - pub refetch_reason: PayloadRefetchReason, + pub payload_feedback: PayloadFeedback, } -/// The reason for having to refetch a data payload in a data notification. +/// The feedback for a given payload. #[derive(Clone, Debug, Eq, PartialEq)] -pub enum PayloadRefetchReason { +pub enum PayloadFeedback { InvalidPayloadData, PayloadTypeIsIncorrect, ProofVerificationFailed, @@ -190,15 +192,23 @@ impl StreamingServiceClient { async fn send_stream_request( &self, client_request: StreamRequest, - ) -> Result { + ) -> Result>, Error> { let mut request_sender = self.request_sender.clone(); let (response_sender, response_receiver) = oneshot::channel(); let request_message = StreamRequestMessage { stream_request: client_request, response_sender, }; - request_sender.send(request_message).await?; + + Ok(response_receiver) + } + + async fn send_request_and_await_response( + &self, + client_request: StreamRequest, + ) -> Result { + let response_receiver = self.send_stream_request(client_request).await?; response_receiver.await? } } @@ -207,7 +217,7 @@ impl StreamingServiceClient { impl DataStreamingClient for StreamingServiceClient { async fn get_all_accounts(&self, version: u64) -> Result { let client_request = StreamRequest::GetAllAccounts(GetAllAccountsRequest { version }); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } async fn get_all_epoch_ending_ledger_infos( @@ -218,7 +228,7 @@ impl DataStreamingClient for StreamingServiceClient { StreamRequest::GetAllEpochEndingLedgerInfos(GetAllEpochEndingLedgerInfosRequest { start_epoch, }); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } async fn get_all_transactions( @@ -234,7 +244,7 @@ impl DataStreamingClient for StreamingServiceClient { max_proof_version, include_events, }); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } async fn get_all_transaction_outputs( @@ -249,20 +259,21 @@ impl DataStreamingClient for StreamingServiceClient { end_version, max_proof_version, }); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } - async fn refetch_notification_payload( + async fn terminate_stream_with_feedback( &self, notification_id: u64, - refetch_reason: PayloadRefetchReason, - ) -> Result { - let client_request = - StreamRequest::RefetchNotificationPayload(RefetchNotificationPayloadRequest { - notification_id, - refetch_reason, - }); - self.send_stream_request(client_request).await + payload_feedback: PayloadFeedback, + ) -> Result<(), Error> { + let client_request = StreamRequest::TerminateStream(TerminateStreamRequest { + notification_id, + payload_feedback, + }); + // We can ignore the receiver as no data will be sent. + let _ = self.send_stream_request(client_request).await?; + Ok(()) } async fn continuously_stream_transactions( @@ -277,7 +288,7 @@ impl DataStreamingClient for StreamingServiceClient { start_epoch, include_events, }); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } async fn continuously_stream_transaction_outputs( @@ -291,7 +302,7 @@ impl DataStreamingClient for StreamingServiceClient { start_epoch, }, ); - self.send_stream_request(client_request).await + self.send_request_and_await_response(client_request).await } } diff --git a/state-sync/state-sync-v2/data-streaming-service/src/streaming_service.rs b/state-sync/state-sync-v2/data-streaming-service/src/streaming_service.rs index 3fd63b8da9..f329259316 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/streaming_service.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/streaming_service.rs @@ -5,9 +5,14 @@ use crate::{ data_stream::{DataStream, DataStreamId, DataStreamListener}, error::Error, logging::{LogEntry, LogEvent, LogSchema}, - streaming_client::{StreamRequestMessage, StreamingServiceListener}, + streaming_client::{ + PayloadFeedback, StreamRequest, StreamRequestMessage, StreamingServiceListener, + TerminateStreamRequest, + }, +}; +use diem_data_client::{ + DiemDataClient, GlobalDataSummary, OptimalChunkSizes, ResponseError, ResponseId, }; -use diem_data_client::{DiemDataClient, GlobalDataSummary, OptimalChunkSizes}; use diem_id_generator::{IdGenerator, U64IdGenerator}; use diem_logger::prelude::*; use futures::StreamExt; @@ -74,7 +79,17 @@ impl DataStreamingService { /// Handles new stream request messages from clients fn handle_stream_request_message(&mut self, request_message: StreamRequestMessage) { - // Process the request message + if let StreamRequest::TerminateStream(request) = request_message.stream_request { + // Process the feedback request + if let Err(error) = self.process_terminate_stream_request(&request) { + error!(LogSchema::new(LogEntry::HandleTerminateRequest) + .event(LogEvent::Error) + .error(&error)); + } + return; + } + + // Process the stream request let response = self.process_new_stream_request(&request_message); if let Err(error) = &response { error!(LogSchema::new(LogEntry::HandleStreamRequest) @@ -93,6 +108,66 @@ impl DataStreamingService { } } + /// Processes a request for terminating the stream that sent a specific + /// notification ID. + fn process_terminate_stream_request( + &mut self, + terminate_request: &TerminateStreamRequest, + ) -> Result<(), Error> { + let notification_id = &terminate_request.notification_id; + + // Find the data stream that sent the notification + let data_stream_ids = self.get_all_data_stream_ids(); + for data_stream_id in &data_stream_ids { + let data_stream = self.get_data_stream(data_stream_id); + if let Some(response_id) = data_stream.get_response_id_for_notification(notification_id) + { + info!(LogSchema::new(LogEntry::HandleTerminateRequest) + .stream_id(*data_stream_id) + .event(LogEvent::Success) + .message(&format!( + "Terminating the stream that sent notification ID: {:?}", + notification_id + ))); + + // Notify the diem data client and delete the stream + self.notify_bad_response( + *data_stream_id, + response_id, + &terminate_request.payload_feedback, + ); + self.data_streams.remove(notification_id); + return Ok(()); + } + } + + panic!( + "Unable to find the stream that sent notification ID: {:?}", + notification_id + ); + } + + /// Notifies the Diem data client of a bad client response + fn notify_bad_response( + &self, + data_stream_id: DataStreamId, + response_id: ResponseId, + payload_feedback: &PayloadFeedback, + ) { + let response_error = extract_response_error(payload_feedback); + + info!(LogSchema::new(LogEntry::HandleTerminateRequest) + .stream_id(data_stream_id) + .event(LogEvent::Success) + .message(&format!( + "Notifying the data client of a bad response. Response id: {:?}, error: {:?}", + response_id, response_error + ))); + + self.diem_data_client + .notify_bad_response(response_id, response_error); + } + /// Creates a new stream and ensures the data for that stream is available fn process_new_stream_request( &mut self, @@ -225,3 +300,13 @@ fn verify_optimal_chunk_sizes(optimal_chunk_sizes: &OptimalChunkSizes) -> Result Ok(()) } } + +/// Transforms the payload feedback into a specific response error that can be +/// sent to the Diem data client. +fn extract_response_error(payload_feedback: &PayloadFeedback) -> ResponseError { + match payload_feedback { + PayloadFeedback::InvalidPayloadData => ResponseError::InvalidData, + PayloadFeedback::PayloadTypeIsIncorrect => ResponseError::InvalidPayloadDataType, + PayloadFeedback::ProofVerificationFailed => ResponseError::ProofVerificationError, + } +} diff --git a/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_client.rs b/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_client.rs index e45cbb2324..cf503efafd 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_client.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_client.rs @@ -9,8 +9,8 @@ use crate::{ new_streaming_service_client_listener_pair, ContinuouslyStreamTransactionOutputsRequest, ContinuouslyStreamTransactionsRequest, DataStreamingClient, GetAllAccountsRequest, GetAllEpochEndingLedgerInfosRequest, GetAllTransactionOutputsRequest, - GetAllTransactionsRequest, PayloadRefetchReason, RefetchNotificationPayloadRequest, - StreamRequest, StreamingServiceListener, + GetAllTransactionsRequest, PayloadFeedback, StreamRequest, StreamingServiceListener, + TerminateStreamRequest, }, tests::utils::initialize_logger, }; @@ -194,29 +194,28 @@ fn test_continuously_stream_transaction_outputs() { } #[test] -fn test_refetch_notification_payloads() { +fn test_terminate_stream() { // Create a new streaming service client and listener let (streaming_service_client, streaming_service_listener) = new_streaming_service_client_listener_pair(); // Note the request we expect to receive on the streaming service side let request_notification_id = 19478; - let request_refetch_reason = PayloadRefetchReason::InvalidPayloadData; - let expected_request = - StreamRequest::RefetchNotificationPayload(RefetchNotificationPayloadRequest { - notification_id: request_notification_id, - refetch_reason: request_refetch_reason.clone(), - }); + let payload_feedback = PayloadFeedback::InvalidPayloadData; + let expected_request = StreamRequest::TerminateStream(TerminateStreamRequest { + notification_id: request_notification_id, + payload_feedback: payload_feedback.clone(), + }); - // Spawn a new server thread to handle any refetch payload requests + // Spawn a new server thread to handle any feedback requests let _handler = spawn_service_and_expect_request(streaming_service_listener, expected_request); - // Send a refetch payload request and verify we get a data stream listener - let response = block_on( + // Provide payload feedback and verify no error is returned + let result = block_on( streaming_service_client - .refetch_notification_payload(request_notification_id, request_refetch_reason), + .terminate_stream_with_feedback(request_notification_id, payload_feedback), ); - assert_ok!(response); + assert_ok!(result); } /// Spawns a new thread that listens to the given streaming service listener and diff --git a/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_service.rs b/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_service.rs index fa7ebf0416..bcd227b914 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_service.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/tests/streaming_service.rs @@ -5,7 +5,7 @@ use crate::{ data_notification::DataPayload, error::Error, streaming_client::{ - new_streaming_service_client_listener_pair, DataStreamingClient, PayloadRefetchReason, + new_streaming_service_client_listener_pair, DataStreamingClient, PayloadFeedback, StreamingServiceClient, }, streaming_service::DataStreamingService, @@ -576,17 +576,80 @@ async fn test_stream_transactions() { assert_matches!(result, Err(Error::DataIsUnavailable(_))); } -#[tokio::test] -async fn test_stream_unsupported() { +#[tokio::test(flavor = "multi_thread")] +#[should_panic(expected = "SelectNextSome polled after terminated")] +async fn test_terminate_stream() { // Create a new streaming client and service let (streaming_client, streaming_service) = create_new_streaming_client_and_service(); tokio::spawn(streaming_service.start_service()); - // Request a refetch notification payload stream and verify it's unsupported - let result = streaming_client - .refetch_notification_payload(0, PayloadRefetchReason::InvalidPayloadData) - .await; - assert_matches!(result, Err(Error::UnsupportedRequestEncountered(_))); + // Request an account stream + let mut stream_listener = streaming_client + .get_all_accounts(MAX_ADVERTISED_ACCOUNTS - 1) + .await + .unwrap(); + + // Fetch the first account notification and then terminate the stream + let mut next_expected_index = 0; + if let Ok(data_notification) = timeout( + Duration::from_secs(MAX_NOTIFICATION_TIMEOUT_SECS), + stream_listener.select_next_some(), + ) + .await + { + match data_notification.data_payload { + DataPayload::AccountStatesWithProof(accounts_with_proof) => { + next_expected_index += accounts_with_proof.account_blobs.len() as u64; + } + data_payload => { + panic!( + "Expected an account ledger info payload, but got: {:?}", + data_payload + ); + } + } + + // Terminate the stream + let result = streaming_client + .terminate_stream_with_feedback( + data_notification.notification_id, + PayloadFeedback::InvalidPayloadData, + ) + .await; + assert_ok!(result); + } else { + panic!("Timed out waiting for a data notification!"); + } + + // Verify the streaming service has removed the stream + loop { + if let Ok(data_notification) = timeout( + Duration::from_secs(MAX_NOTIFICATION_TIMEOUT_SECS), + stream_listener.select_next_some(), + ) + .await + { + match data_notification.data_payload { + DataPayload::AccountStatesWithProof(accounts_with_proof) => { + next_expected_index += accounts_with_proof.account_blobs.len() as u64; + } + DataPayload::EndOfStream => { + panic!("The stream should have terminated!"); + } + data_payload => { + panic!( + "Expected an account ledger info payload, but got: {:?}", + data_payload + ); + } + } + } else if next_expected_index >= TOTAL_NUM_ACCOUNTS { + panic!( + "The stream should have terminated! Next expected index: {:?}", + next_expected_index + ); + } + } } fn create_new_streaming_client_and_service() -> ( diff --git a/state-sync/state-sync-v2/data-streaming-service/src/tests/utils.rs b/state-sync/state-sync-v2/data-streaming-service/src/tests/utils.rs index ed27bca2b5..d6bab0b96d 100644 --- a/state-sync/state-sync-v2/data-streaming-service/src/tests/utils.rs +++ b/state-sync/state-sync-v2/data-streaming-service/src/tests/utils.rs @@ -209,7 +209,7 @@ impl DiemDataClient for MockDiemDataClient { } fn notify_bad_response(&self, _response_id: u64, _response_error: ResponseError) { - // TODO(joshlind): implement this + // TODO(joshlind): update me to handle some score emulation! } }