Skip to content

Commit

Permalink
[DI-75][Indexer] Create Token V2 data models and convert token v1 (ap…
Browse files Browse the repository at this point in the history
…tos-labs#7359)

* Add token v2 initial indexing

* lint

* remove index for performance

* remove token activities v2

* Remove fungible asset indexing

* [indexer] token v2 property map

(cherry picked from commit 96e0116)

* Apply property map parsing

* handle burn nft

---------

Co-authored-by: Aaron Gao <[email protected]>
  • Loading branch information
bowenyang007 and lightmark authored May 8, 2023
1 parent 9af7ef7 commit 68efb02
Show file tree
Hide file tree
Showing 21 changed files with 2,850 additions and 39 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
-- This file should undo anything in `up.sql`
DROP TABLE IF EXISTS objects;
DROP INDEX IF EXISTS o_owner_idx;
DROP INDEX IF EXISTS o_object_skh_idx;
DROP INDEX IF EXISTS o_skh_idx;
DROP INDEX IF EXISTS o_insat_idx;
DROP TABLE IF EXISTS current_objects;
DROP INDEX IF EXISTS co_owner_idx;
DROP INDEX IF EXISTS co_object_skh_idx;
DROP INDEX IF EXISTS co_skh_idx;
DROP INDEX IF EXISTS co_insat_idx;
ALTER TABLE move_resources DROP COLUMN IF EXISTS state_key_hash;
DROP TABLE IF EXISTS token_ownerships_v2;
DROP INDEX IF EXISTS to2_id_index;
DROP INDEX IF EXISTS to2_owner_index;
DROP INDEX IF EXISTS to2_insat_index;
DROP TABLE IF EXISTS current_token_ownerships_v2;
DROP INDEX IF EXISTS curr_to2_owner_index;
DROP INDEX IF EXISTS curr_to2_wa_index;
DROP INDEX IF EXISTS curr_to2_insat_index;
DROP TABLE IF EXISTS collections_v2;
DROP INDEX IF EXISTS col2_id_index;
DROP INDEX IF EXISTS col2_crea_cn_index;
DROP INDEX IF EXISTS col2_insat_index;
DROP TABLE IF EXISTS current_collections_v2;
DROP INDEX IF EXISTS cur_col2_crea_cn_index;
DROP INDEX IF EXISTS cur_col2_insat_index;
DROP TABLE IF EXISTS token_datas_v2;
DROP INDEX IF EXISTS td2_id_index;
DROP INDEX IF EXISTS td2_cid_name_index;
DROP INDEX IF EXISTS td2_insat_index;
DROP TABLE IF EXISTS current_token_datas_v2;
DROP INDEX IF EXISTS cur_td2_cid_name_index;
DROP INDEX IF EXISTS cur_td2_insat_index;
ALTER TABLE current_token_pending_claims DROP COLUMN IF EXISTS token_data_id;
ALTER TABLE current_token_pending_claims DROP COLUMN IF EXISTS collection_id;
170 changes: 170 additions & 0 deletions crates/indexer/migrations/2023-04-28-053048_object_token_v2/up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
-- Your SQL goes here
-- objects, basically normalizing ObjectCore
CREATE TABLE IF NOT EXISTS objects (
transaction_version BIGINT NOT NULL,
write_set_change_index BIGINT NOT NULL,
object_address VARCHAR(66) NOT NULL,
owner_address VARCHAR(66),
state_key_hash VARCHAR(66) NOT NULL,
guid_creation_num NUMERIC,
allow_ungated_transfer BOOLEAN,
is_deleted BOOLEAN NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW(),
-- constraints
PRIMARY KEY (transaction_version, write_set_change_index)
);
CREATE INDEX IF NOT EXISTS o_owner_idx ON objects (owner_address);
CREATE INDEX IF NOT EXISTS o_object_skh_idx ON objects (object_address, state_key_hash);
CREATE INDEX IF NOT EXISTS o_skh_idx ON objects (state_key_hash);
CREATE INDEX IF NOT EXISTS o_insat_idx ON objects (inserted_at);
-- latest instance of objects
CREATE TABLE IF NOT EXISTS current_objects (
object_address VARCHAR(66) UNIQUE PRIMARY KEY NOT NULL,
owner_address VARCHAR(66) NOT NULL,
state_key_hash VARCHAR(66) NOT NULL,
allow_ungated_transfer BOOLEAN NOT NULL,
last_guid_creation_num NUMERIC NOT NULL,
last_transaction_version BIGINT NOT NULL,
is_deleted BOOLEAN NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS co_owner_idx ON current_objects (owner_address);
CREATE INDEX IF NOT EXISTS co_object_skh_idx ON current_objects (object_address, state_key_hash);
CREATE INDEX IF NOT EXISTS co_skh_idx ON current_objects (state_key_hash);
CREATE INDEX IF NOT EXISTS co_insat_idx ON current_objects (inserted_at);
-- Add this so that we can find resource groups by their state_key_hash
ALTER TABLE move_resources
ADD COLUMN IF NOT EXISTS state_key_hash VARCHAR(66) NOT NULL DEFAULT '';
-- NFT stuff
-- tracks who owns tokens
CREATE TABLE IF NOT EXISTS token_ownerships_v2 (
transaction_version BIGINT NOT NULL,
write_set_change_index BIGINT NOT NULL,
token_data_id VARCHAR(66) NOT NULL,
property_version_v1 NUMERIC NOT NULL,
owner_address VARCHAR(66),
storage_id VARCHAR(66) NOT NULL,
amount NUMERIC NOT NULL,
table_type_v1 VARCHAR(66),
token_properties_mutated_v1 JSONB,
is_soulbound_v2 BOOLEAN,
token_standard VARCHAR(10) NOT NULL,
is_fungible_v2 BOOLEAN,
transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW(),
PRIMARY KEY (transaction_version, write_set_change_index)
);
CREATE INDEX IF NOT EXISTS to2_id_index ON token_ownerships_v2 (token_data_id);
CREATE INDEX IF NOT EXISTS to2_owner_index ON token_ownerships_v2 (owner_address);
CREATE INDEX IF NOT EXISTS to2_insat_index ON token_ownerships_v2 (inserted_at);
CREATE TABLE IF NOT EXISTS current_token_ownerships_v2 (
token_data_id VARCHAR(66) NOT NULL,
property_version_v1 NUMERIC NOT NULL,
owner_address VARCHAR(66) NOT NULL,
storage_id VARCHAR(66) NOT NULL,
amount NUMERIC NOT NULL,
table_type_v1 VARCHAR(66),
token_properties_mutated_v1 JSONB,
is_soulbound_v2 BOOLEAN,
token_standard VARCHAR(10) NOT NULL,
is_fungible_v2 BOOLEAN,
last_transaction_version BIGINT NOT NULL,
last_transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW(),
PRIMARY KEY (
token_data_id,
property_version_v1,
owner_address,
storage_id
)
);
CREATE INDEX IF NOT EXISTS curr_to2_owner_index ON current_token_ownerships_v2 (owner_address);
CREATE INDEX IF NOT EXISTS curr_to2_wa_index ON current_token_ownerships_v2 (storage_id);
CREATE INDEX IF NOT EXISTS curr_to2_insat_index ON current_token_ownerships_v2 (inserted_at);
-- tracks collections
CREATE TABLE IF NOT EXISTS collections_v2 (
transaction_version BIGINT NOT NULL,
write_set_change_index BIGINT NOT NULL,
collection_id VARCHAR(66) NOT NULL,
creator_address VARCHAR(66) NOT NULL,
collection_name VARCHAR(128) NOT NULL,
description TEXT NOT NULL,
uri VARCHAR(512) NOT NULL,
current_supply NUMERIC NOT NULL,
max_supply NUMERIC,
total_minted_v2 NUMERIC,
mutable_description BOOLEAN,
mutable_uri BOOLEAN,
table_handle_v1 VARCHAR(66),
token_standard VARCHAR(10) NOT NULL,
transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW(),
PRIMARY KEY (transaction_version, write_set_change_index)
);
CREATE INDEX IF NOT EXISTS col2_id_index ON collections_v2 (collection_id);
CREATE INDEX IF NOT EXISTS col2_crea_cn_index ON collections_v2 (creator_address, collection_name);
CREATE INDEX IF NOT EXISTS col2_insat_index ON collections_v2 (inserted_at);
CREATE TABLE IF NOT EXISTS current_collections_v2 (
collection_id VARCHAR(66) UNIQUE PRIMARY KEY NOT NULL,
creator_address VARCHAR(66) NOT NULL,
collection_name VARCHAR(128) NOT NULL,
description TEXT NOT NULL,
uri VARCHAR(512) NOT NULL,
current_supply NUMERIC NOT NULL,
max_supply NUMERIC,
total_minted_v2 NUMERIC,
mutable_description BOOLEAN,
mutable_uri BOOLEAN,
table_handle_v1 VARCHAR(66),
token_standard VARCHAR(10) NOT NULL,
last_transaction_version BIGINT NOT NULL,
last_transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS cur_col2_crea_cn_index ON current_collections_v2 (creator_address, collection_name);
CREATE INDEX IF NOT EXISTS cur_col2_insat_index ON current_collections_v2 (inserted_at);
-- tracks token metadata
CREATE TABLE IF NOT EXISTS token_datas_v2 (
transaction_version BIGINT NOT NULL,
write_set_change_index BIGINT NOT NULL,
token_data_id VARCHAR(66) NOT NULL,
collection_id VARCHAR(66) NOT NULL,
token_name VARCHAR(128) NOT NULL,
maximum NUMERIC,
supply NUMERIC NOT NULL,
largest_property_version_v1 NUMERIC,
token_uri VARCHAR(512) NOT NULL,
token_properties JSONB NOT NULL,
description TEXT NOT NULL,
token_standard VARCHAR(10) NOT NULL,
is_fungible_v2 BOOLEAN,
transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW(),
PRIMARY KEY (transaction_version, write_set_change_index)
);
CREATE INDEX IF NOT EXISTS td2_id_index ON token_datas_v2 (token_data_id);
CREATE INDEX IF NOT EXISTS td2_cid_name_index ON token_datas_v2 (collection_id, token_name);
CREATE INDEX IF NOT EXISTS td2_insat_index ON token_datas_v2 (inserted_at);
CREATE TABLE IF NOT EXISTS current_token_datas_v2 (
token_data_id VARCHAR(66) UNIQUE PRIMARY KEY NOT NULL,
collection_id VARCHAR(66) NOT NULL,
token_name VARCHAR(128) NOT NULL,
maximum NUMERIC,
supply NUMERIC NOT NULL,
largest_property_version_v1 NUMERIC,
token_uri VARCHAR(512) NOT NULL,
description TEXT NOT NULL,
token_properties JSONB NOT NULL,
token_standard VARCHAR(10) NOT NULL,
is_fungible_v2 BOOLEAN,
last_transaction_version BIGINT NOT NULL,
last_transaction_timestamp TIMESTAMP NOT NULL,
inserted_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS cur_td2_cid_name_index ON current_token_datas_v2 (collection_id, token_name);
CREATE INDEX IF NOT EXISTS cur_td2_insat_index ON current_token_datas_v2 (inserted_at);
-- Add ID (with 0x prefix)
ALTER TABLE current_token_pending_claims
ADD COLUMN IF NOT EXISTS token_data_id VARCHAR(66) NOT NULL DEFAULT '';
ALTER TABLE current_token_pending_claims
ADD COLUMN IF NOT EXISTS collection_id VARCHAR(66) NOT NULL DEFAULT '';
2 changes: 1 addition & 1 deletion crates/indexer/src/models/coin_models/coin_activities.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const GAS_FEE_EVENT: &str = "0x1::aptos_coin::GasFeeEvent";
// We will never have a negative number on chain so this will avoid collision in postgres
const BURN_GAS_EVENT_CREATION_NUM: i64 = -1;
const BURN_GAS_EVENT_INDEX: i64 = -1;
const MAX_ENTRY_FUNCTION_LENGTH: usize = 100;
pub const MAX_ENTRY_FUNCTION_LENGTH: usize = 100;

type OwnerAddress = String;
type CoinType = String;
Expand Down
2 changes: 1 addition & 1 deletion crates/indexer/src/models/coin_models/coin_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ pub struct IntegerWrapperResource {

impl IntegerWrapperResource {
/// In case we do want to track supply
pub fn _get_supply(&self) -> Option<BigDecimal> {
pub fn get_supply(&self) -> Option<BigDecimal> {
self.vec.get(0).map(|inner| inner.value.clone())
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/indexer/src/models/coin_models/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ pub mod coin_activities;
pub mod coin_balances;
pub mod coin_infos;
pub mod coin_supply;
mod coin_utils;
pub mod coin_utils;
1 change: 1 addition & 0 deletions crates/indexer/src/models/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@ pub mod stake_models;
pub mod token_models;
pub mod transactions;
pub mod user_transactions;
pub mod v2_objects;
pub mod write_set_changes;
3 changes: 3 additions & 0 deletions crates/indexer/src/models/move_resources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ pub struct MoveResource {
pub generic_type_params: Option<serde_json::Value>,
pub data: Option<serde_json::Value>,
pub is_deleted: bool,
pub state_key_hash: String,
}

pub struct MoveStructTag {
Expand Down Expand Up @@ -50,6 +51,7 @@ impl MoveResource {
generic_type_params: parsed_data.generic_type_params,
data: Some(serde_json::to_value(&write_resource.data.data).unwrap()),
is_deleted: false,
state_key_hash: standardize_address(write_resource.state_key_hash.as_str()),
}
}

Expand All @@ -71,6 +73,7 @@ impl MoveResource {
generic_type_params: parsed_data.generic_type_params,
data: None,
is_deleted: true,
state_key_hash: standardize_address(delete_resource.state_key_hash.as_str()),
}
}

Expand Down
50 changes: 50 additions & 0 deletions crates/indexer/src/models/property_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,53 @@ impl PropertyMap {
serde_json::to_value(map).unwrap()
}
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct TokenObjectPropertyValue {
value: String,
typ: u8,
}

pub fn create_token_object_property_value(
typ: u8,
value: String,
) -> Result<TokenObjectPropertyValue> {
Ok(TokenObjectPropertyValue {
value: util::convert_bcs_hex_new(typ, value.clone()).unwrap_or(value),
typ,
})
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct TokenObjectPropertyMap {
data: HashMap<String, TokenObjectPropertyValue>,
}

impl TokenObjectPropertyMap {
/// Deserializes PropertyValue from bcs encoded json
pub fn from_bcs_encode_str(val: Value) -> Option<Value> {
let mut pm = TokenObjectPropertyMap {
data: HashMap::new(),
};
let records: &Vec<Value> = val.get("data")?.as_array()?;
for entry in records {
let key = entry.get("key")?.as_str()?;
let val = entry.get("value")?.get("value")?.as_str()?;
let typ = entry.get("value")?.get("type")?.as_u64()?;
let pv = create_token_object_property_value(typ as u8, val.to_string()).ok()?;
pm.data.insert(key.to_string(), pv);
}
Some(Self::to_flat_json_new(pm))
}

/// Flattens PropertyMap which can't be easily consumable by downstream.
/// For example: Object {"data": Object {"creation_time_sec": Object {"value": String("1666125588")}}}
/// becomes Object {"creation_time_sec": "1666125588"}
fn to_flat_json_new(val: TokenObjectPropertyMap) -> Value {
let mut map = HashMap::new();
for (k, v) in val.data {
map.insert(k, v.value);
}
serde_json::to_value(map).unwrap()
}
}
4 changes: 2 additions & 2 deletions crates/indexer/src/models/token_models/collection_datas.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ use diesel::{prelude::*, ExpressionMethods};
use field_count::FieldCount;
use serde::{Deserialize, Serialize};

const QUERY_RETRIES: u32 = 5;
const QUERY_RETRY_DELAY_MS: u64 = 500;
pub const QUERY_RETRIES: u32 = 5;
pub const QUERY_RETRY_DELAY_MS: u64 = 500;
#[derive(Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)]
#[diesel(primary_key(collection_data_id_hash, transaction_version))]
#[diesel(table_name = collection_datas)]
Expand Down
4 changes: 4 additions & 0 deletions crates/indexer/src/models/token_models/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,7 @@ pub mod token_datas;
pub mod token_ownerships;
pub mod token_utils;
pub mod tokens;
pub mod v2_collections;
pub mod v2_token_datas;
pub mod v2_token_ownerships;
pub mod v2_token_utils;
Loading

0 comments on commit 68efb02

Please sign in to comment.