Skip to content

Commit

Permalink
fix change host state rollback and maintain fail when create vm
Browse files Browse the repository at this point in the history 
Resolve ZSTAC-8790
  • Loading branch information
@MaJin1996
MaJin1996 committed Jan 21, 2018
1 parent 79afe10 commit c9e8c3b
Show file tree
Hide file tree
Showing 9 changed files with 339 additions and 158 deletions.
4 changes: 2 additions & 2 deletions compute/src/main/java/org/zstack/compute/host/HostApiInterceptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ private void validate(APIChangeHostStateMsg msg){
.select(HostVO_.status)
.eq(HostVO_.uuid,msg.getHostUuid())
.findValue();
if (hostStatus == HostStatus.Connecting && msg.getStateEvent().equals(HostStateEvent.maintain.toString())){
throw new ApiMessageInterceptionException(operr("can not maintain host[uuid:%s]which is connecting", msg.getHostUuid()));
if (hostStatus != HostStatus.Connected && msg.getStateEvent().equals(HostStateEvent.maintain.toString())){
throw new ApiMessageInterceptionException(operr("can not maintain host[uuid:%s]which is not Connected", msg.getHostUuid()));
}
}
}
97 changes: 44 additions & 53 deletions compute/src/main/java/org/zstack/compute/host/HostBase.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.zstack.compute.host;

import org.apache.commons.collections.list.SynchronizedList;
import org.springframework.beans.factory.annotation.Autowire;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Configurable;
Expand Down Expand Up @@ -45,10 +46,7 @@
import org.zstack.utils.function.ForEachFunction;
import org.zstack.utils.logging.CLogger;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.TimeUnit;

import static org.zstack.core.Platform.operr;
Expand Down Expand Up @@ -212,26 +210,26 @@ public void run(final FlowTrigger trigger, Map data) {
}
}

final List<MigrateVmMsg> msgs = new ArrayList<MigrateVmMsg>();
for (String uuid : vmUuids) {
new While<>(vmUuids).step((vmUuid, compl) -> {
MigrateVmMsg msg = new MigrateVmMsg();
msg.setVmInstanceUuid(uuid);
bus.makeTargetServiceIdByResourceUuid(msg, VmInstanceConstant.SERVICE_ID, uuid);
msgs.add(msg);
}

bus.send(msgs, migrateQuantity, new CloudBusListCallBack(trigger) {
@Override
public void run(List<MessageReply> replies) {
for (MessageReply reply : replies) {
msg.setVmInstanceUuid(vmUuid);
bus.makeTargetServiceIdByResourceUuid(msg, VmInstanceConstant.SERVICE_ID, vmUuid);
bus.send(msg, new CloudBusCallBack(compl) {
@Override
public void run(MessageReply reply) {
if (!reply.isSuccess()) {
MigrateVmMsg msg = msgs.get(replies.indexOf(reply));
logger.warn(String.format("failed to migrate vm[uuid:%s] on host[uuid:%s, name:%s, ip:%s], will try stopping it. %s",
msg.getVmInstanceUuid(), self.getUuid(), self.getName(), self.getManagementIp(), reply.getError()));
vmFailedToMigrate.add(msg.getVmInstanceUuid());
}
compl.done();
}
});
}, migrateQuantity).run(new NoErrorCompletion() {
@Override
public void done() {
if (!vmFailedToMigrate.isEmpty()) {
logger.warn(String.format("failed to migrate vm[uuids:%s] on host[uuid:%s, name:%s, ip:%s], will try stopping it.",
vmFailedToMigrate, self.getUuid(), self.getName(), self.getManagementIp()));
}

trigger.next();
}
});
Expand All @@ -253,50 +251,44 @@ public void run(List<MessageReply> replies) {

@Override
public void run(FlowTrigger trigger, Map data) {
List<String> vmUuids = new ArrayList<String>();
vmUuids.addAll(vmFailedToMigrate);
vmUuids = CollectionUtils.removeDuplicateFromList(vmUuids);

if (vmUuids.isEmpty()) {
if (vmFailedToMigrate.isEmpty()) {
trigger.next();
return;
}

List<String> vmUuids = Q.New(VmInstanceVO.class).select(VmInstanceVO_.uuid)
.in(VmInstanceVO_.uuid, vmFailedToMigrate).listValues();
stopFailedToMigrateVms(vmUuids, trigger);
}

private void stopFailedToMigrateVms(List<String> vmUuids, final FlowTrigger trigger) {
final List<StopVmInstanceMsg> msgs = new ArrayList<StopVmInstanceMsg>();
for (String vmUuid : vmUuids) {
List<ErrorCode> errors = Collections.synchronizedList(new ArrayList<>());
List<String> vmFailedToStop = Collections.synchronizedList(new ArrayList<>());
new While<>(vmUuids).step((vmUuid, coml) -> {
StopVmInstanceMsg msg = new StopVmInstanceMsg();
msg.setVmInstanceUuid(vmUuid);
bus.makeTargetServiceIdByResourceUuid(msg, VmInstanceConstant.SERVICE_ID, vmUuid);
msgs.add(msg);
}

bus.send(msgs, quantity, new CloudBusListCallBack(trigger) {
@Override
public void run(List<MessageReply> replies) {
StringBuilder sb = new StringBuilder();
boolean success = true;
for (MessageReply r : replies) {
if (!r.isSuccess()) {
StopVmInstanceMsg msg = msgs.get(replies.indexOf(r));
String err = String.format("\nfailed to stop vm[uuid:%s] on host[uuid:%s, name:%s, ip:%s], %s",
msg.getVmInstanceUuid(), self.getUuid(), self.getName(), self.getManagementIp(), r.getError());
sb.append(err);
success = false;
bus.send(msg, new CloudBusCallBack(coml) {
@Override
public void run(MessageReply reply) {
logger.debug(String.format("stop vm[uuid:%s] call back", vmUuid));
if (!reply.isSuccess() && dbf.isExist(vmUuid, VmInstanceVO.class)) {
errors.add(reply.getError());
vmFailedToStop.add(vmUuid);
}
coml.done();
}

if (!success) {
logger.warn(sb.toString());
}

if (success || HostGlobalConfig.IGNORE_ERROR_ON_MAINTENANCE_MODE.value(Boolean.class)) {
});
}, quantity).run(new NoErrorCompletion() {
@Override
public void done() {
if (errors.isEmpty() || HostGlobalConfig.IGNORE_ERROR_ON_MAINTENANCE_MODE.value(Boolean.class)) {
trigger.next();
} else {
trigger.fail(operr(sb.toString()));
trigger.fail(errf.instantiateErrorCode(SysErrors.OPERATION_ERROR,
String.format("failed to stop vm[uuids:%s] on host[uuid:%s, name:%s, ip:%s]",
vmFailedToStop, self.getUuid(), self.getName(), self.getManagementIp()),
errors));
}
}
});
Expand Down Expand Up @@ -453,10 +445,6 @@ private void done(SyncTaskChain chain) {
public void run(final SyncTaskChain chain) {
final APIChangeHostStateEvent evt = new APIChangeHostStateEvent(msg.getId());
HostStateEvent stateEvent = HostStateEvent.valueOf(msg.getStateEvent());

if (self.getStatus() == HostStatus.Disconnected && stateEvent == HostStateEvent.maintain) {
throw new ApiMessageInterceptionException(operr("cannot change the state of Disconnected host into Maintenance "));
}
stateEvent = stateEvent == HostStateEvent.maintain ? HostStateEvent.preMaintain : stateEvent;
try {
extpEmitter.preChange(self, stateEvent);
Expand All @@ -468,6 +456,7 @@ public void run(final SyncTaskChain chain) {
}

if (HostStateEvent.preMaintain == stateEvent) {
HostState originState = self.getState();
changeState(HostStateEvent.preMaintain);
maintenanceHook(new Completion(msg, chain) {
@Override
Expand All @@ -481,7 +470,9 @@ public void success() {
@Override
public void fail(ErrorCode errorCode) {
evt.setError(errf.instantiateErrorCode(HostErrors.UNABLE_TO_ENTER_MAINTENANCE_MODE, errorCode.getDetails(), errorCode));
changeState(HostStateEvent.enable);
HostStateEvent rollbackEvent = dbf.reload(self).getState().getTargetStateDrivenEvent(originState);
DebugUtils.Assert(rollbackEvent != null, "rollbackEvent not found!");
changeState(rollbackEvent);
bus.publish(evt);
done(chain);
}
Expand Down
6 changes: 3 additions & 3 deletions conf/i18n.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1435,9 +1435,9 @@
"fileName": "src/main/java/org/zstack/compute/host/HostApiInterceptor.java"
},
{
"raw": "can not maintain host[uuid:%s]which is connecting",
"en_US": "can not maintain host[uuid:{0}]which is connecting",
"zh_CN": "不能对Connecting状态的物理机[uuid:{0}]进行维护操作",
"raw": "can not maintain host[uuid:%s]which is not Connected",
"en_US": "can not maintain host[uuid:{0}]which is not Connected",
"zh_CN": "只能对已连接状态的物理机[uuid:{0}]进行维护操作",
"arguments": [
"msg.getHostUuid()"
],
Expand Down
2 changes: 1 addition & 1 deletion conf/i18n/messages_en_US.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ managementIp[%s]\ is\ neither\ an\ IPv4\ address\ nor\ a\ valid\ hostname = mana

# at: src/main/java/org/zstack/compute/host/HostApiInterceptor.java:94
# args: msg.getHostUuid()
can\ not\ maintain\ host[uuid\:%s]which\ is\ connecting = can not maintain host[uuid:{0}]which is connecting
can\ not\ maintain\ host[uuid\:%s]which\ is\ not\ Connected = can not maintain host[uuid:{0}]which is not Connected

# at: src/main/java/org/zstack/compute/host/HostBase.java:104
# args: self.getUuid(),self.getName(),self.getState()
Expand Down
2 changes: 1 addition & 1 deletion conf/i18n/messages_zh_CN.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ managementIp[%s]\ is\ neither\ an\ IPv4\ address\ nor\ a\ valid\ hostname = 管

# at: src/main/java/org/zstack/compute/host/HostApiInterceptor.java:94
# args: msg.getHostUuid()
can\ not\ maintain\ host[uuid\:%s]which\ is\ connecting = 不能对Connecting状态的物理机[uuid:{0}]进行维护操作
can\ not\ maintain\ host[uuid\:%s]which\ is\ not\ Connected = 只能对已连接状态的物理机[uuid:{0}]进行维护操作

# at: src/main/java/org/zstack/compute/host/HostBase.java:104
# args: self.getUuid(),self.getName(),self.getState()
Expand Down
3 changes: 3 additions & 0 deletions header/src/main/java/org/zstack/header/host/APIChangeHostStateMsg.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ public ApiNotification __notification__() {
return new ApiNotification() {
@Override
public void after(APIEvent evt) {
if (!evt.isSuccess()) {
return;
}
ntfy("Changed the state to %s", ((APIChangeHostStateEvent)evt).getInventory().getState())
.resource(uuid, HostVO.class.getSimpleName())
.messageAndEvent(that, evt).done();
Expand Down
4 changes: 4 additions & 0 deletions header/src/main/java/org/zstack/header/host/HostState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,8 @@ public HostState nextState(HostStateEvent event) {

return tran.nextState;
}

public HostStateEvent getTargetStateDrivenEvent(HostState targetState) {
return transactionMap.values().stream().filter(it -> it.nextState == targetState).findFirst().map(it -> it.event).orElse(null);
}
}
98 changes: 0 additions & 98 deletions ...t/groovy/org/zstack/test/integration/kvm/host/ChangeDisconncetedHostToMaintainCase.groovy
View file

This file was deleted.

Loading

0 comments on commit c9e8c3b

Please sign in to comment.