Optimization for start child workflow task (cadence-workflow#4315)

* Optimization for start child workflow task

common/domain/attrValidator.go

@@ -92,7 +92,7 @@ func (d *AttrValidatorImpl) validateDomainReplicationConfigForGlobalDomain(
 	replicationConfig *persistence.DomainReplicationConfig,
 ) error {
 	// TODO: https://github.com/uber/cadence/issues/4345 add checking for "pending active" as well
-	// Right now we only have checking if clusters to remove are "current active cluster" in this method. 
+	// Right now we only have checking if clusters to remove are "current active cluster" in this method.
 	// However, there could be edge cases that a cluster is in "pending active" state during graceful failover.
 	// It's better to do this check so that people won't make mistake.
 	// However, this is not critical -- even this happens, they can add the active cluster back

service/history/task/transfer_active_task_executor.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2020 Uber Technologies, Inc.
+// Copyright (c) 2021 Uber Technologies, Inc.
+// Portions of the Software are attributed to Copyright (c) 2021 Temporal Technologies Inc.
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -644,16 +645,18 @@ func (t *transferActiveTaskExecutor) processStartChildExecution(
 
 	// ChildExecution already started, just create DecisionTask and complete transfer task
 	if childInfo.StartedID != common.EmptyEventID {
-		// entity not exist error is checked and ignored in HandleErr() method in task.go
+		// NOTE: do not access anything related mutable state after this lock release
+		// release the context lock since we no longer need mutable state builder and
+		// the rest of logic is making RPC call, which takes time.
+		release(nil)
 		return createFirstDecisionTask(
 			ctx,
 			t.historyClient,
 			task.TargetDomainID,
 			&types.WorkflowExecution{
 				WorkflowID: childInfo.StartedWorkflowID,
 				RunID:      childInfo.StartedRunID,
-			},
-		)
+			})
 	}
 
 	attributes := initiatedEvent.StartChildWorkflowExecutionInitiatedEventAttributes
@@ -688,6 +691,10 @@ func (t *transferActiveTaskExecutor) processStartChildExecution(
 		return err
 	}
 
+	// NOTE: do not access anything related mutable state after this lock release
+	// release the context lock since we no longer need mutable state builder and
+	// the rest of logic is making RPC call, which takes time.
+	release(nil)
 	// Finally create first decision task for Child execution so it is really started
 	// entity not exist error is checked and ignored in HandleErr() method in task.go
 	return createFirstDecisionTask(