WIP; fix partition table for non-default size; more supplied checkpoi…

…nts fixes

apps/arweave/src/ar_coordination.erl

@@ -284,7 +284,7 @@ handle_cast({reset_mining_session, _MiningSession}, State) ->
 
 handle_cast({compute_h2, Peer, H2Materials}, State) ->
 	ar_mining_server:remote_compute_h2(Peer, H2Materials),
-	{_Diff, _Addr, _H0, _PartitionNumber, _PartitionUpperBound, ReqList} = H2Materials,
+	{_Diff, _Addr, _H0, _PartitionNumber, _PartitionUpperBound, _Seed, _NextSeed, _StartIntervalNumber, _StepNumber, _NonceLimiterOutput, _SuppliedCheckpoints, ReqList} = H2Materials,
 	OldStat = maps:get(Peer, State#state.peer_io_stat, #peer_io_stat{}),
 	H1Count = length(ReqList),
 	NewStat = OldStat#peer_io_stat{
@@ -297,9 +297,9 @@ handle_cast({compute_h2, Peer, H2Materials}, State) ->
 	},
 	{noreply, NewState};
 
-handle_cast({computed_h2, {Diff, Addr, H0, H1, Nonce, PartitionNumber, PartitionUpperBound, PoA2, H2, Preimage, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, Peer}}, State) ->
+handle_cast({computed_h2, {Diff, Addr, H0, H1, Nonce, PartitionNumber, PartitionUpperBound, PoA2, H2, Preimage, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints, Peer}}, State) ->
 	io:format("DEBUG computed_h2~n"),
-	ar_http_iface_client:cm_h2_send(Peer, {Diff, Addr, H0, H1, Nonce, PartitionNumber, PartitionUpperBound, PoA2, H2, Preimage, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput}),
+	ar_http_iface_client:cm_h2_send(Peer, {Diff, Addr, H0, H1, Nonce, PartitionNumber, PartitionUpperBound, PoA2, H2, Preimage, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints}),
 	OldStat = maps:get(Peer, State#state.peer_io_stat, #peer_io_stat{}),
 	NewStat = OldStat#peer_io_stat{
 		h2_out_counter = OldStat#peer_io_stat.h2_out_counter + 1

apps/arweave/src/ar_http_iface_middleware.erl

@@ -2988,16 +2988,38 @@ handle_get_vdf2(Req, Call) ->
 
 handle_coordinated_mining_partition_table(Req) ->
 	{ok, Config} = application:get_env(arweave, config),
-	Table = lists:map(
-		fun	({BucketSize, Bucket, {spora_2_6, Addr}}) ->
-			{[
-				{bucket, Bucket},
-				{bucketsize, BucketSize},
-				{addr, ar_util:encode(Addr)}
-			]}
-		end,
-		Config#config.storage_modules
-	),
+	Partition_dict = lists:foldl(fun(Module, Dict) ->
+		{BucketSize, Bucket, {spora_2_6, Addr}} = Module,
+		MinPartitionId = Bucket*BucketSize div ?PARTITION_SIZE,
+		MaxPartitionId = (Bucket+1)*BucketSize div ?PARTITION_SIZE,
+		PartitionList = lists:seq(MinPartitionId, MaxPartitionId),
+		lists:foldl(fun(PartitionId, Dict2) ->
+			AddrDict = maps:get(PartitionId, Dict2, #{}),
+			AddrDict2 = maps:put(Addr, true, AddrDict),
+			maps:put(PartitionId, AddrDict2, Dict2)
+		end, Dict, PartitionList)
+	end, #{}, Config#config.storage_modules),
+	Right_bound = lists:max(lists:map(fun({BucketSize, Bucket, {spora_2_6, _Addr}}) ->
+		(Bucket+1)*BucketSize
+	end, Config#config.storage_modules)),
+	Right_bound_partition_id = Right_bound div ?PARTITION_SIZE,
+	CheckPartitionList = lists:seq(0, Right_bound_partition_id),
+	Table = lists:foldr(fun(PartitionId, Acc) ->
+		case maps:find(PartitionId, Partition_dict) of
+			{ok, AddrDict} ->
+				maps:fold(fun(Addr, _, Acc2) ->
+					NewEntity = {[
+						{bucket, PartitionId},
+						{bucketsize, ?PARTITION_SIZE},
+						{addr, ar_util:encode(Addr)}
+						% TODO range start, end for less requests (better check before send)
+					]},
+					[NewEntity | Acc2]
+				end, Acc, AddrDict);
+			_ ->
+				Acc
+		end
+	end, [], CheckPartitionList),
 	{200, #{}, ar_serialize:jsonify(Table), Req}.
 
 read_complete_body(Req, Pid) ->

apps/arweave/src/ar_mining_server.erl

@@ -285,7 +285,7 @@ handle_cast({pause_performance_reports, Time}, State) ->
 			pause_performance_reports_timeout = Timeout }};
 
 handle_cast({remote_compute_h2, Peer, H2Materials}, State) ->
-	{Diff, Addr, H0, PartitionNumber, PartitionUpperBound, NonceLimiterOutput,
+	{Diff, Addr, H0, PartitionNumber, PartitionUpperBound, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints,
 			ReqList} = H2Materials,
 	{_RecallRange1Start, RecallRange2Start} = ar_block:get_recall_range(H0,
 			PartitionNumber, PartitionUpperBound),
@@ -300,7 +300,7 @@ handle_cast({remote_compute_h2, Peer, H2Materials}, State) ->
 			reserve_cache_space(),
 			CorrelationRef = {PartitionNumber2, PartitionUpperBound, make_ref()},
 			Session = {remote, Diff, Addr, H0, PartitionNumber, PartitionUpperBound,
-					RecallRange2Start, NonceLimiterOutput, ReqList, Peer},
+					RecallRange2Start, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints, ReqList, Peer},
 			Thread ! {remote_read_recall_range2, self(), Session, CorrelationRef}
 	end,
 	{noreply, State};
@@ -475,6 +475,7 @@ handle_info({remote_io_thread_recall_range2_chunk,
 		Session}}, State) ->
 	%% Prevent an accidental pattern match of _H0, _PartitionNumber.
 	{remote, _Diff, _Addr, _H0_, _PartitionNumber_, _PartitionUpperBound, _RecallByte2Start,
+			_Seed, _NextSeed, _StartIntervalNumber, _StepNumber, _NonceLimiterOutput2, _SuppliedCheckpoints,
 			ReqList, _Peer } = Session,
 	#state{ hashing_threads = Threads } = State,
 	%% The accumulator is in fact the un-accumulator here.
@@ -616,6 +617,7 @@ io_thread(PartitionNumber, ReplicaID, StoreID, SessionRef) ->
 			io_thread(PartitionNumber, ReplicaID, StoreID, SessionRef);
 		{remote_read_recall_range2, From, Session, CorrelationRef} ->
 			{remote, _Diff, Addr, H0, PartitionNumber2, _PartitionUpperBound, RecallRangeStart,
+					_Seed, _NextSeed, _StartIntervalNumber, _StepNumber, _NonceLimiterOutput, _SuppliedCheckpoints,
 					_ReqList, _Peer} = Session,
 			case ReplicaID of
 				Addr ->
@@ -854,7 +856,8 @@ hashing_thread(SessionRef) ->
 			%% Important: here we make http requests inside the hashing thread
 			%% to reduce the latency.
 			{remote, Diff, ReplicaID, H0, PartitionNumber, PartitionUpperBound,
-					_RecallByte2Start, NonceLimiterOutput, _ReqList, Peer } = Session,
+					_RecallByte2Start, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints,
+					_ReqList, Peer } = Session,
 			{H2, Preimage2} = ar_block:compute_h2(H1, Chunk2, H0),
 			case binary:decode_unsigned(H2, big) > Diff of
 				true ->
@@ -885,9 +888,13 @@ hashing_thread(SessionRef) ->
 									{mining_address, ar_util:encode(ReplicaID)}]),
 							ok;
 						_ ->
+							[{_, TipNonceLimiterInfo}] = ets:lookup(node_state, nonce_limiter_info),
+							#nonce_limiter_info{ next_seed = PrevNextSeed,
+								global_step_number = PrevStepNumber } = TipNonceLimiterInfo,
+							SuppliedCheckpoints = ar_nonce_limiter:get_checkpoints(PrevStepNumber, StepNumber, PrevNextSeed),
 							ar_coordination:computed_h2({Diff, ReplicaID, H0, H1, Nonce,
 									PartitionNumber, PartitionUpperBound, PoA2, H2, Preimage2,
-									NonceLimiterOutput, Peer})
+									Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints, Peer})
 					end;
 				false ->
 					ok

apps/arweave/src/ar_serialize.erl

@@ -1573,9 +1573,11 @@ json_map_to_remote_h2_materials(JSON) ->
 		Nonce = maps:get(<<"nonce">>, JsonElement),
 		{H1, Nonce}
 	end, maps:get(<<"req_list">>, JSON, [])),
-	{Diff, Addr, H0, PartitionNumber, PartitionUpperBound, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, ReqList}.
+	SuppliedCheckpointsEncoded = maps:get(<<"vdf_checkpoints">>, JSON),
+	SuppliedCheckpoints = parse_checkpoints(ar_util:decode(SuppliedCheckpointsEncoded), 1),
+	{Diff, Addr, H0, PartitionNumber, PartitionUpperBound, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints, ReqList}.
 
-remote_h2_materials_to_json_map({Diff, Addr, H0, PartitionNumber, PartitionUpperBound, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, ReqList}) ->
+remote_h2_materials_to_json_map({Diff, Addr, H0, PartitionNumber, PartitionUpperBound, Seed, NextSeed, StartIntervalNumber, StepNumber, NonceLimiterOutput, SuppliedCheckpoints, ReqList}) ->
 	ReqList2 = lists:map(fun ({H1, Nonce}) ->
 		{[
 			{h1, ar_util:encode(H1)},
@@ -1594,6 +1596,7 @@ remote_h2_materials_to_json_map({Diff, Addr, H0, PartitionNumber, PartitionUpper
 		{start_interval_number, integer_to_binary(StartIntervalNumber)},
 		{step_number, integer_to_binary(StepNumber)},
 		{nonce_limiter_output, ar_util:encode(NonceLimiterOutput)},
+		{vdf_checkpoints, ar_util:encode(iolist_to_binary(SuppliedCheckpoints))},
 		{req_list, ReqList2}
 	].