Skip to content

Commit

Permalink
PRT - fix e2e race condition and invalid behavior (lavanet#1085)
Browse files Browse the repository at this point in the history
* fix init_e2e race condition and invalid behavior

* better stability

* adding more info to debug

* test adjustment

* fix e2e

* fix

* fix e2e adding another block wait because of begin block event order

* adding delay because github action is so slow.

* fix a bug related to qos excellence report.

* fix optimizer race condition on unitests

* upgrade lavajs to v0.32.4

* increase protocol patch version to 32.5

* using rlock instead

* fix division of zero

* fix lint

* fixed race in init_e2e

* changed sync score to one instead of 0 in excellence due to 0 being ideal value and 1 being 0 contribution
  • Loading branch information
ranlavanet authored Dec 31, 2023
1 parent 88bc936 commit abc96ed
Show file tree
Hide file tree
Showing 10 changed files with 150 additions and 64 deletions.
67 changes: 26 additions & 41 deletions cookbook/plans/test_plans/temporary-add.json
Original file line number Diff line number Diff line change
@@ -1,45 +1,30 @@
{
"proposal": {
"title": "Add temporary to-delete plan proposal",
"description": "A proposal of a temporary to-delete plan",
"plans": [
{
"index": "to_delete_plan",
"description": "This plan has no restrictions",
"type": "rpc",
"price": {
"denom": "ulava",
"amount": "100000"
},
"annual_discount_percentage": 20,
"allow_overuse": true,
"overuse_rate": 2,
"plan_policy": {
"chain_policies": [
{
"chain_id": "LAV1",
"apis": [
]
},
{
"chain_id": "ETH1",
"apis": [
"eth_blockNumber",
"eth_accounts"
]
}
],
"geolocation_profile": "AU",
"total_cu_limit": 1000000,
"epoch_cu_limit": 100000,
"max_providers_to_pair": 3,
"selected_providers_mode": "MIXED",
"selected_providers": [
"lava@1wvn4slrf2r7cm92fnqdhvl3x470944uev92squ"
]
}
}
]
"title": "to_delete_plan",
"description": "to_delete_plan",
"plans": [
{
"index": "to_delete_plan",
"description": "to_delete_plan",
"type": "rpc",
"price": {
"denom": "ulava",
"amount": "10000"
},
"annual_discount_percentage": 20,
"allow_overuse": false,
"overuse_rate": 0,
"plan_policy": {
"chain_policies": [],
"geolocation_profile": "GL",
"total_cu_limit": 100000,
"epoch_cu_limit": 50,
"max_providers_to_pair": 5,
"selected_providers_mode": "ALLOWED",
"selected_providers": []
}
}
]
},
"deposit": "10000000ulava"
}
}
3 changes: 2 additions & 1 deletion ecosystem/lavajs/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@lavanet/lavajs",
"version": "0.22.5",
"version": "0.32.4",
"description": "lavajs",
"author": "Lava Network",
"homepage": "https://github.com/lavanet/lava/tree/main/ecosystem/lavajs#readme",
Expand All @@ -17,6 +17,7 @@
],
"scripts": {
"e2e-setup": "./scripts/build_lavajs.sh -s",
"build_lava_js": "./scripts/build_lavajs.sh -s",
"init": "./scripts/build_lavajs.sh -s",
"clean:mjs": "rimraf mjs",
"clean:dist": "rimraf dist",
Expand Down
2 changes: 1 addition & 1 deletion protocol/lavasession/consumer_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ func (cs *SingleConsumerSession) CalculateExpectedLatency(timeoutGivenToRelay ti
// cs should be locked here to use this method, returns the computed qos or zero if last qos is nil or failed to compute.
func (cs *SingleConsumerSession) getQosComputedResultOrZero() sdk.Dec {
if cs.QoSInfo.LastExcellenceQoSReport != nil {
qosComputed, errComputing := cs.QoSInfo.LastExcellenceQoSReport.ComputeQoS()
qosComputed, errComputing := cs.QoSInfo.LastExcellenceQoSReport.ComputeQoSExcellence()
if errComputing == nil { // if we failed to compute the qos will be 0 so this provider wont be picked to return the error in case we get it
return qosComputed
}
Expand Down
11 changes: 10 additions & 1 deletion protocol/provideroptimizer/provider_optimizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ type ConcurrentBlockStore struct {
Block uint64
}

type cacheInf interface {
Get(key interface{}) (interface{}, bool)
Set(key, value interface{}, cost int64) bool
}

type ProviderOptimizer struct {
strategy Strategy
providersStorage *ristretto.Cache
providersStorage cacheInf
providerRelayStats *ristretto.Cache // used to decide on the half time of the decay
averageBlockTime time.Duration
baseWorldLatency time.Duration
Expand Down Expand Up @@ -467,6 +472,10 @@ func (po *ProviderOptimizer) GetExcellenceQoSReportForProvider(providerAddress s
precision := WANTED_PRECISION
latencyScore := turnFloatToDec(providerData.Latency.Num/providerData.Latency.Denom, precision)
syncScore := turnFloatToDec(providerData.Sync.Num/providerData.Sync.Denom, precision)
// if our sync score is un initialized due to lack of providers
if syncScore.IsZero() {
syncScore = sdk.OneDec()
}
availabilityScore := turnFloatToDec(providerData.Availability.Num/providerData.Availability.Denom, precision)
ret := &pairingtypes.QualityOfServiceReport{
Latency: latencyScore,
Expand Down
24 changes: 22 additions & 2 deletions protocol/provideroptimizer/provider_optimizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package provideroptimizer

import (
"strconv"
"sync"
"testing"
"time"

Expand All @@ -16,6 +17,25 @@ const (
TEST_BASE_WORLD_LATENCY = 150 * time.Millisecond
)

type providerOptimizerSyncCache struct {
value map[interface{}]interface{}
lock sync.RWMutex
}

func (posc *providerOptimizerSyncCache) Get(key interface{}) (interface{}, bool) {
posc.lock.RLock()
defer posc.lock.RUnlock()
ret, ok := posc.value[key]
return ret, ok
}

func (posc *providerOptimizerSyncCache) Set(key, value interface{}, cost int64) bool {
posc.lock.Lock()
defer posc.lock.Unlock()
posc.value[key] = value
return true
}

func setupProviderOptimizer(maxProvidersCount int) *ProviderOptimizer {
averageBlockTIme := TEST_AVERAGE_BLOCK_TIME
baseWorldLatency := TEST_BASE_WORLD_LATENCY
Expand Down Expand Up @@ -214,6 +234,7 @@ func TestProviderOptimizerAvailabilityBlockError(t *testing.T) {
pertrubationPercentage := 0.0
syncBlock := uint64(requestBlock)
chosenIndex := rand.Intn(providersCount)

for i := range providersGen.providersAddresses {
time.Sleep(4 * time.Millisecond)
// give all providers a worse availability score
Expand Down Expand Up @@ -242,12 +263,12 @@ func TestProviderOptimizerUpdatingLatency(t *testing.T) {
requestCU := uint64(10)
requestBlock := int64(1000)
syncBlock := uint64(requestBlock)
providerOptimizer.providersStorage = &providerOptimizerSyncCache{value: map[interface{}]interface{}{}}
// in this test we are repeatedly adding better results, and latency score should improve
for i := 0; i < 10; i++ {
providerData, _ := providerOptimizer.getProviderData(providerAddress)
currentLatencyScore := providerOptimizer.calculateLatencyScore(providerData, requestCU, requestBlock)
providerOptimizer.AppendProbeRelayData(providerAddress, TEST_BASE_WORLD_LATENCY, true)
time.Sleep(4 * time.Millisecond)
providerData, found := providerOptimizer.getProviderData(providerAddress)
require.True(t, found)
newLatencyScore := providerOptimizer.calculateLatencyScore(providerData, requestCU, requestBlock)
Expand All @@ -258,7 +279,6 @@ func TestProviderOptimizerUpdatingLatency(t *testing.T) {
providerData, _ := providerOptimizer.getProviderData(providerAddress)
currentLatencyScore := providerOptimizer.calculateLatencyScore(providerData, requestCU, requestBlock)
providerOptimizer.AppendRelayData(providerAddress, TEST_BASE_WORLD_LATENCY, false, requestCU, syncBlock)
time.Sleep(4 * time.Millisecond)
providerData, found := providerOptimizer.getProviderData(providerAddress)
require.True(t, found)
newLatencyScore := providerOptimizer.calculateLatencyScore(providerData, requestCU, requestBlock)
Expand Down
12 changes: 6 additions & 6 deletions scripts/init_chain.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ if [ "$1" == "debug" ]; then
data=$(cat "$path$genesis" \
| jq '.app_state.gov.params.min_deposit[0].denom = "ulava"' \
| jq '.app_state.gov.params.min_deposit[0].amount = "100"' \
| jq '.app_state.gov.params.voting_period = "3s"' \
| jq '.app_state.gov.params.expedited_voting_period = "1s"' \
| jq '.app_state.gov.params.voting_period = "4s"' \
| jq '.app_state.gov.params.expedited_voting_period = "3s"' \
| jq '.app_state.gov.params.expedited_min_deposit[0].denom = "ulava"' \
| jq '.app_state.gov.params.expedited_min_deposit[0].amount = "200"' \
| jq '.app_state.gov.params.expedited_threshold = "0.67"' \
Expand All @@ -43,17 +43,17 @@ else
data=$(cat "$path$genesis" \
| jq '.app_state.gov.params.min_deposit[0].denom = "ulava"' \
| jq '.app_state.gov.params.min_deposit[0].amount = "100"' \
| jq '.app_state.gov.params.voting_period = "3s"' \
| jq '.app_state.gov.params.expedited_voting_period = "1s"' \
| jq '.app_state.gov.params.voting_period = "4s"' \
| jq '.app_state.gov.params.expedited_voting_period = "3s"' \
| jq '.app_state.gov.params.expedited_min_deposit[0].denom = "ulava"' \
| jq '.app_state.gov.params.expedited_min_deposit[0].amount = "200"' \
| jq '.app_state.gov.params.expedited_threshold = "0.67"' \
| jq '.app_state.mint.params.mint_denom = "ulava"' \
| jq '.app_state.mint.params.mint_denom = "ulava"' \
| jq '.app_state.staking.params.bond_denom = "ulava"' \
| jq '.app_state.crisis.constant_fee.denom = "ulava"' \
| jq '.app_state.downtime.params.downtime_duration = "10s"' \
| jq '.app_state.downtime.params.epoch_duration = "20s"' \
| jq '.app_state.downtime.params.downtime_duration = "6s"' \
| jq '.app_state.downtime.params.epoch_duration = "10s"' \
)
fi

Expand Down
36 changes: 25 additions & 11 deletions scripts/init_e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,23 @@ GASPRICE="0.000000001ulava"
echo ---- Specs proposal ----
lavad tx gov submit-legacy-proposal spec-add ./cookbook/specs/spec_add_ethereum.json,./cookbook/specs/spec_add_ibc.json,./cookbook/specs/spec_add_cosmossdk.json,./cookbook/specs/spec_add_lava.json --lava-dev-test -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov deposit 1 100ulava -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov vote 1 yes -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE

sleep 6 # need to sleep because plan policies need the specs when setting chain policies verifications

# Plans proposal
echo ---- Plans proposal ----
wait_next_block
lavad tx gov submit-legacy-proposal plans-add ./cookbook/plans/test_plans/default.json,./cookbook/plans/test_plans/emergency-mode.json,./cookbook/plans/test_plans/temporary-add.json -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov deposit 2 100ulava -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov vote 2 yes -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
sleep 6

# Plan removal (of one)
echo ---- Plans removal ----
wait_next_block
# delete plan that deletes "temporary add" plan
lavad tx gov submit-legacy-proposal plans-del ./cookbook/plans/test_plans/temporary-del.json -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov deposit 3 1ulava -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
wait_next_block
lavad tx gov vote 3 yes -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE

STAKE="500000000000ulava"
Expand Down Expand Up @@ -62,14 +56,18 @@ lavad tx subscription buy "EmergencyModePlan" -y --from user5 --gas-adjustment "
# Test plan upgrade
echo ---- Subscription plan upgrade ----
wait_next_block
lavad tx subscription buy "DefaultPlan" -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
# test we have the plan active.
plan_index=$(lavad q subscription current $(lavad keys show user1 -a) | yq .sub.plan_index)
if [ "$plan_index" != "EmergencyModePlan" ]; then "echo subscription ${user1addr}: wrong plan index $plane_index instead of EmergencyModePlan"; exit 1; fi
if [ "$plan_index" != "EmergencyModePlan" ]; then "echo subscription ${user1addr}: wrong plan index $plan_index .sub.plan_index doesn't contain EmergencyModePlan"; exit 1; fi
# buy the upgraded subscription
lavad tx subscription buy "DefaultPlan" -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE

# wait for the new subscription to take effect (1 epoch + 1 block as changes happen to subscription module after epochstorage module on the begin block events)
wait_next_block # wait block is here in case 1 block before epoch change we commit but the effect happens only on the epoch change meaning we didn't really wait an epoch for the changes to take effect.
sleep_until_next_epoch

# validate the new subscription is the default plan and not emergency mode plan.
plan_index=$(lavad q subscription current $(lavad keys show user1 -a) | yq .sub.plan_index)
if [ "$plan_index" != "DefaultPlan" ]; then "echo subscription ${user1addr}: wrong plan index $plane_index instead of DefaultPlan"; exit 1; fi
if [ "$plan_index" != "DefaultPlan" ]; then "echo subscription ${user1addr}: wrong plan index $plan_index .sub.plan_index doesn't contain DefaultPlan"; exit 1; fi

user3addr=$(lavad keys show user3 -a)

Expand All @@ -93,4 +91,20 @@ sleep_until_next_epoch
count=$(lavad q subscription list-projects ${user3addr} | grep "lava@" | wc -l)
if [ "$count" -ne 2 ]; then "echo subscription ${user3addr}: wrong project count $count instead of 2"; exit 1; fi


# validate deleted plan is removed.
# Fetch the plans list
plans_list=$(lavad q plan list | yq .plans_info)

# Check if "to_delete_plan" exists
if echo "$plans_list" | grep -q '"index": "to_delete_plan"'; then
echo "Index 'to_delete_plan' exists."
exit 1 # fail test.
else
echo "Index 'to_delete_plan' was removed successfully validation passed."
fi



# the end

9 changes: 9 additions & 0 deletions x/pairing/types/QualityOfServiceReport.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,12 @@ func (qos *QualityOfServiceReport) ComputeQoS() (sdk.Dec, error) {

return qos.Availability.Mul(qos.Sync).Mul(qos.Latency).ApproxRoot(3)
}

func (qos *QualityOfServiceReport) ComputeQoSExcellence() (sdk.Dec, error) {
if qos.Availability.LTE(sdk.ZeroDec()) ||
qos.Latency.LTE(sdk.ZeroDec()) ||
qos.Sync.LTE(sdk.ZeroDec()) {
return sdk.ZeroDec(), fmt.Errorf("QoS excellence scores is below 0")
}
return qos.Availability.Quo(qos.Sync).Quo(qos.Latency).ApproxRoot(3)
}
48 changes: 48 additions & 0 deletions x/pairing/types/QualityOfServiceReport_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package types

import (
"testing"

sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/stretchr/testify/require"
)

func TestQosReport(t *testing.T) {
qos1 := &QualityOfServiceReport{
Latency: sdk.MustNewDecFromStr("1.5"),
Availability: sdk.MustNewDecFromStr("1"),
Sync: sdk.MustNewDecFromStr("0.1"),
}
qos2 := &QualityOfServiceReport{
Latency: sdk.MustNewDecFromStr("0.2"),
Availability: sdk.MustNewDecFromStr("1"),
Sync: sdk.MustNewDecFromStr("0.1"),
}
qos3 := &QualityOfServiceReport{
Latency: sdk.MustNewDecFromStr("0.1"),
Availability: sdk.MustNewDecFromStr("1"),
Sync: sdk.MustNewDecFromStr("0.5"),
}
qos4 := &QualityOfServiceReport{
Latency: sdk.MustNewDecFromStr("0.1"),
Availability: sdk.MustNewDecFromStr("0.5"),
Sync: sdk.MustNewDecFromStr("0.5"),
}

qos1Res, errQos1 := qos1.ComputeQoSExcellence()
qos2Res, errQos2 := qos2.ComputeQoSExcellence()
qos3Res, errQos3 := qos3.ComputeQoSExcellence()
qos4Res, errQos4 := qos4.ComputeQoSExcellence()
require.NoError(t, errQos1)
require.NoError(t, errQos2)
require.NoError(t, errQos3)
require.NoError(t, errQos4)
require.True(t, qos1Res.LT(qos2Res))
require.True(t, qos1Res.LT(qos3Res))
require.True(t, qos1Res.LT(qos4Res))

require.True(t, qos2Res.GT(qos3Res))
require.True(t, qos2Res.GT(qos4Res))

require.True(t, qos4Res.LT(qos3Res))
}
2 changes: 1 addition & 1 deletion x/protocol/types/params.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
var _ paramtypes.ParamSet = (*Params)(nil)

const (
TARGET_VERSION = "0.32.4"
TARGET_VERSION = "0.32.5"
MIN_VERSION = "0.30.1"
)

Expand Down

0 comments on commit abc96ed

Please sign in to comment.