diff --git a/config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties b/config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties index cb1b14c85ace..ac1e1b97419d 100644 --- a/config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties +++ b/config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties @@ -1188,6 +1188,12 @@ # Default: 30 #session.cluster.minSecsAfterRebuild=30 +# When a node is in shutdown should we redirect to another working node? +# When a node is in shutdown and this is true we pick another node in the cluster and redirect to that node by setting +# the cookie to point to that node, when this is false we just delete the cookie and let the load balancer decide +# Default: true +# cluster.redirect.random.node=false + # ######################################################################## # SERVLET CONTAINER diff --git a/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterNode.java b/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterNode.java new file mode 100644 index 000000000000..ea308dc325fc --- /dev/null +++ b/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterNode.java @@ -0,0 +1,28 @@ +package org.sakaiproject.cluster.api; + +import java.util.Date; + +/** + * This holds information about a node in the cluster. + */ +public interface ClusterNode { + + /** + * Gets the status of a node. + * @return The current node status, or {@link org.sakaiproject.cluster.api.ClusterService.Status#UNKNOWN} + * if the status isn't known. + */ + ClusterService.Status getStatus(); + + /** + * Gets the server ID of a node. + * @return The server ID. + */ + String getServerId(); + + /** + * Gets when the status of the node was last updated. + * @return The date when the status of the node was last updated. + */ + Date getUpdated(); +} diff --git a/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterService.java b/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterService.java index 76440fa0941e..6ec0d13b8763 100644 --- a/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterService.java +++ b/kernel/api/src/main/java/org/sakaiproject/cluster/api/ClusterService.java @@ -87,7 +87,7 @@ enum Status * Get the statuses of the servers in the cluster. * @return A Map of the servers with the value being the server status. */ - Map getServerStatus(); + Map getServerStatus(); /** * Marks a server as being closed. This prevents new sessions from being started. diff --git a/kernel/api/src/main/java/org/sakaiproject/util/RequestFilter.java b/kernel/api/src/main/java/org/sakaiproject/util/RequestFilter.java index d9856ee4b562..8a49b75b4e4b 100644 --- a/kernel/api/src/main/java/org/sakaiproject/util/RequestFilter.java +++ b/kernel/api/src/main/java/org/sakaiproject/util/RequestFilter.java @@ -28,6 +28,9 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.sakaiproject.cluster.api.ClusterNode; +import org.sakaiproject.cluster.api.ClusterService; +import org.sakaiproject.cluster.api.ClusterService.Status; import org.sakaiproject.component.api.ServerConfigurationService; import org.sakaiproject.component.cover.ComponentManager; import org.sakaiproject.event.api.UsageSession; @@ -176,6 +179,10 @@ public class RequestFilter implements Filter /** The tools allowed as lti provider **/ protected static final String SAKAI_BLTI_PROVIDER_TOOLS = "basiclti.provider.allowedtools"; + + /** The name of the Skaia property to say we should redirect to another node when in shutdown */ + protected static final String SAKAI_CLUSTER_REDIRECT_RANDOM = "cluster.redirect.random.node"; + /** Our log (commons). */ private static Log M_log = LogFactory.getLog(RequestFilter.class); /** If true, we deliver the Sakai end user enterprise id as the remote user in each request. */ @@ -205,6 +212,8 @@ public class RequestFilter implements Filter protected boolean m_displayModJkWarning = true; + protected boolean m_redirectRandomNode = true; + /** Default is to abort further upload processing if the max is exceeded. */ protected boolean m_uploadContinue = false; @@ -555,7 +564,7 @@ protected void closingRedirect(HttpServletRequest req, HttpServletResponse res) // We could check that we aren't in a redirect loop here, but if the load balancer doesn't know that // a node is no longer responding to new sessions it may still be sending it new clients, and so after // a couple of redirects it should hop off this node. - String value = ""; + String value = getRedirectNode(); // set the cookie Cookie c = new Cookie(cookieName, value); c.setPath("/"); @@ -579,6 +588,43 @@ protected void closingRedirect(HttpServletRequest req, HttpServletResponse res) res.sendRedirect(url.toString()); } + /** + * This looks to find a node to redirect to or if it can't find one it just empties the cookie + * so the load balancer chooses. + * @return The cookie value for a different node. + */ + protected String getRedirectNode() { + if (m_redirectRandomNode) { + ClusterService clusterService = (ClusterService) ComponentManager.get(ClusterService.class); + Map nodes = clusterService.getServerStatus(); + // There may be more than one node listed for each node ID, just list the latest ones. + Map latestNodes = new HashMap<>(); + for (ClusterNode node: nodes.values()) { + ClusterNode latest = latestNodes.get(node.getServerId()); + if (latest == null || latest.getUpdated().after(node.getUpdated())) { + latestNodes.put(node.getServerId(), node); + } + } + // This node shouldn't ever be included but it's better safe than sorry. + latestNodes.remove(System.getProperty(SAKAI_SERVERID)); + // Remove all the non-running servers. + List activeServers = new ArrayList<>(latestNodes.size()); + for (ClusterNode node : latestNodes.values()) { + if (Status.RUNNING.equals(node.getStatus())) { + activeServers.add(node.getServerId()); + } + } + // Pick a random remaining server if we have one. + if (!(activeServers.isEmpty())) { + Random random = new Random(); + int i = random.nextInt(activeServers.size()); + String serverId = activeServers.get(i); + return DOT + serverId; + } + } + return ""; + } + /** * If any of these files exist, delete them. * @@ -606,6 +652,7 @@ public void init(FilterConfig filterConfig) throws ServletException // sakai.properties settings to system properties - see SakaiPropertyPromoter() ServerConfigurationService configService = org.sakaiproject.component.cover.ServerConfigurationService.getInstance(); + // knl-640 appUrl = configService.getString("serverUrl", null); chsDomain = configService.getString("content.chs.serverName", null); @@ -772,10 +819,12 @@ else if ("tool".equalsIgnoreCase(s)) // retrieve option to enable or disable cookie HttpOnly m_cookieHttpOnly = configService.getBoolean(SAKAI_COOKIE_HTTP_ONLY, true); - m_UACompatible = configService.getString(SAKAI_UA_COMPATIBLE,null); + m_UACompatible = configService.getString(SAKAI_UA_COMPATIBLE, null); isLTIProviderAllowed = (configService.getString(SAKAI_BLTI_PROVIDER_TOOLS,null)!=null); + m_redirectRandomNode = configService.getBoolean(SAKAI_CLUSTER_REDIRECT_RANDOM, true); + } /** diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/ClusterNodeImpl.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/ClusterNodeImpl.java new file mode 100644 index 000000000000..3b647c644782 --- /dev/null +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/ClusterNodeImpl.java @@ -0,0 +1,38 @@ +package org.sakaiproject.cluster.impl; + +import org.sakaiproject.cluster.api.ClusterNode; +import org.sakaiproject.cluster.api.ClusterService; + +import java.util.Date; + +/** + * Simple immutable implementation of ClusterNode. + */ +public class ClusterNodeImpl implements ClusterNode { + + private final String serverId; + private final ClusterService.Status status; + private final Date updated; + + + public ClusterNodeImpl(String serverId, ClusterService.Status status, Date updated) { + this.serverId = serverId; + this.status = status; + this.updated = updated; + } + + @Override + public ClusterService.Status getStatus() { + return status; + } + + @Override + public String getServerId() { + return serverId; + } + + @Override + public Date getUpdated() { + return updated; + } +} diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterService.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterService.java index 6a479501552b..e41a0d07db37 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterService.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterService.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.sakaiproject.cluster.api.ClusterNode; import org.sakaiproject.cluster.api.ClusterService; import org.sakaiproject.component.api.ServerConfigurationService; import org.sakaiproject.component.cover.ComponentManager; @@ -293,10 +294,10 @@ public List getServers() } @Override - public Map getServerStatus() + public Map getServerStatus() { String statement = clusterServiceSql.getListServerStatusSql(); - final Map servers = new HashMap(); + final Map servers = new HashMap<>(); m_sqlService.dbRead(statement, null, new SqlReader() { @Override @@ -304,10 +305,12 @@ public Object readSqlResultRecord(ResultSet result) throws SqlReaderFinishedExce { try { + String serverInstanceId = result.getString("SERVER_ID_INSTANCE"); String serverId = result.getString("SERVER_ID"); + Date updateTime = result.getTimestamp("UPDATE_TIME"); Status status = parseStatus(result.getString("STATUS")); - // Happy to put null into status? or should we convert to UNKNOWN? - servers.put(serverId, status); + ClusterNode node = new ClusterNodeImpl(serverId, status, updateTime); + servers.put(serverInstanceId, node); } catch (SQLException e) { @@ -317,14 +320,15 @@ public Object readSqlResultRecord(ResultSet result) throws SqlReaderFinishedExce } }); // Always override DB status with memory version. - Status dbStatus = servers.put(m_serverConfigurationService.getServerIdInstance(), status); + ClusterNode dbStatus = servers.put(m_serverConfigurationService.getServerIdInstance(), + new ClusterNodeImpl(m_serverConfigurationService.getServerId(), status, new Date())); if (dbStatus == null) { M_log.warn("Failed to find ourselves in the cluster: "+ m_serverConfigurationService.getServerIdInstance()); } - else if (!status.equals(dbStatus)) + else if (!status.equals(dbStatus.getStatus())) { - M_log.warn("In memory status ("+ status+ ") different to DB ("+ dbStatus+ ")"); + M_log.warn("In memory status ("+ status+ ") different to DB ("+ dbStatus.getStatus()+ ")"); } return servers; } @@ -388,9 +392,10 @@ public void start() // register in the cluster table String statement = clusterServiceSql.getInsertServerSql(); - Object fields[] = new Object[2]; + Object fields[] = new Object[3]; fields[0] = m_serverConfigurationService.getServerIdInstance(); fields[1] = Status.STARTING.toString(); + fields[2] = m_serverConfigurationService.getServerId(); boolean ok = m_sqlService.dbWrite(statement, fields); if (!ok) @@ -591,9 +596,10 @@ private void updateOurStatus(String serverIdInstance) M_log.warn("run(): server has been closed in cluster table, reopened: " + serverIdInstance); statement = clusterServiceSql.getInsertServerSql(); - fields = new Object[2]; + fields = new Object[3]; fields[0] = serverIdInstance; fields[1] = status; + fields[2] = m_serverConfigurationService.getServerId(); boolean ok = m_sqlService.dbWrite(statement, fields); if (!ok) { @@ -606,9 +612,10 @@ private void updateOurStatus(String serverIdInstance) { // register that this app server is alive and well statement = clusterServiceSql.getUpdateServerSql(); - fields = new Object[2]; + fields = new Object[3]; fields[0] = status; - fields[1] = serverIdInstance; + fields[1] = m_serverConfigurationService.getServerId(); + fields[2] = serverIdInstance; boolean ok = m_sqlService.dbWrite(statement, fields); if (!ok) { @@ -641,7 +648,7 @@ public Object readSqlResultRecord(ResultSet result) throws SqlReaderFinishedExce private Status parseStatus(String statusString) { - Status status = null; + Status status = Status.UNKNOWN; if (statusString != null) { try diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlDefault.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlDefault.java index 15598666b0ec..ec6fd131dc94 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlDefault.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlDefault.java @@ -51,7 +51,7 @@ public String getOrphanedLockSessionsSql() */ public String getDeleteServerSql() { - return "delete from SAKAI_CLUSTER where SERVER_ID = ?"; + return "delete from SAKAI_CLUSTER where SERVER_ID_INSTANCE = ?"; } /** @@ -59,7 +59,7 @@ public String getDeleteServerSql() */ public String getInsertServerSql() { - return "insert into SAKAI_CLUSTER (SERVER_ID,UPDATE_TIME, STATUS) values (?, " + sqlTimestamp() + ", ?)"; + return "insert into SAKAI_CLUSTER (SERVER_ID_INSTANCE, UPDATE_TIME, STATUS, SERVER_ID) values (?, " + sqlTimestamp() + ", ?, ?)"; } /** @@ -69,7 +69,7 @@ public String getInsertServerSql() */ public String getListExpiredServers(long timeout) { - return "select SERVER_ID from SAKAI_CLUSTER where SERVER_ID != ? and DATEDIFF('ss', UPDATE_TIME, CURRENT_TIMESTAMP) >= " + timeout; + return "select SERVER_ID_INSTANCE from SAKAI_CLUSTER where SERVER_ID_INSTANCE != ? and DATEDIFF('ss', UPDATE_TIME, CURRENT_TIMESTAMP) >= " + timeout; } /** @@ -77,7 +77,7 @@ public String getListExpiredServers(long timeout) */ public String getListServersSql() { - return "select SERVER_ID from SAKAI_CLUSTER order by SERVER_ID asc"; + return "select SERVER_ID_INSTANCE from SAKAI_CLUSTER order by SERVER_ID_INSTANCE asc"; } /** @@ -85,7 +85,7 @@ public String getListServersSql() */ public String getReadServerSql() { - return "select SERVER_ID, STATUS from SAKAI_CLUSTER where SERVER_ID = ?"; + return "select SERVER_ID_INSTANCE, STATUS from SAKAI_CLUSTER where SERVER_ID_INSTANCE = ?"; } /** @@ -93,13 +93,13 @@ public String getReadServerSql() */ public String getUpdateServerSql() { - return "update SAKAI_CLUSTER set UPDATE_TIME = " + sqlTimestamp() + ", STATUS = ? where SERVER_ID = ?"; + return "update SAKAI_CLUSTER set UPDATE_TIME = " + sqlTimestamp() + ", STATUS = ?, SERVER_ID = ? where SERVER_ID_INSTANCE = ?"; } @Override public String getListServerStatusSql() { - return "SELECT SERVER_ID, STATUS FROM SAKAI_CLUSTER ORDER BY SERVER_ID ASC"; + return "SELECT SERVER_ID_INSTANCE, STATUS, SERVER_ID, UPDATE_TIME FROM SAKAI_CLUSTER ORDER BY SERVER_ID_INSTANCE ASC"; } /** diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlMySql.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlMySql.java index 5d55ba3abc91..75d0d06c71a8 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlMySql.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlMySql.java @@ -34,7 +34,7 @@ public class SakaiClusterServiceSqlMySql extends SakaiClusterServiceSqlDefault */ public String getListExpiredServers(long timeout) { - return "select SERVER_ID from SAKAI_CLUSTER where SERVER_ID != ? and UPDATE_TIME < CURRENT_TIMESTAMP() - INTERVAL " + timeout + " SECOND"; + return "select SERVER_ID_INSTANCE from SAKAI_CLUSTER where SERVER_ID_INSTANCE != ? and UPDATE_TIME < CURRENT_TIMESTAMP() - INTERVAL " + timeout + " SECOND"; } /** diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlOracle.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlOracle.java index 115135de44bd..5f55541d4b18 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlOracle.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/cluster/impl/SakaiClusterServiceSqlOracle.java @@ -34,7 +34,7 @@ public class SakaiClusterServiceSqlOracle extends SakaiClusterServiceSqlDefault */ public String getListExpiredServers(long timeout) { - return "select SERVER_ID from SAKAI_CLUSTER where SERVER_ID != ? and UPDATE_TIME < (CURRENT_TIMESTAMP - " + return "select SERVER_ID_INSTANCE from SAKAI_CLUSTER where SERVER_ID_INSTANCE != ? and UPDATE_TIME < (CURRENT_TIMESTAMP - " + ((float) timeout / (float) (60 * 60 * 24)) + " )"; } } diff --git a/kernel/kernel-impl/src/main/sql/hsqldb/sakai_cluster.sql b/kernel/kernel-impl/src/main/sql/hsqldb/sakai_cluster.sql index 0daec50e4ab7..dd8bd36f1749 100644 --- a/kernel/kernel-impl/src/main/sql/hsqldb/sakai_cluster.sql +++ b/kernel/kernel-impl/src/main/sql/hsqldb/sakai_cluster.sql @@ -4,7 +4,9 @@ CREATE TABLE SAKAI_CLUSTER ( - SERVER_ID VARCHAR (64), + SERVER_ID_INSTANCE VARCHAR (64), UPDATE_TIME DATETIME, - PRIMARY KEY (SERVER_ID) + STATUS VARCHAR(8), -- No enums for us here. + SERVER_ID VARCHAR(64) + PRIMARY KEY (SERVER_ID_INSTANCE) ); diff --git a/kernel/kernel-impl/src/main/sql/mysql/sakai_cluster.sql b/kernel/kernel-impl/src/main/sql/mysql/sakai_cluster.sql index 8c775d8d262f..91a5599901b8 100644 --- a/kernel/kernel-impl/src/main/sql/mysql/sakai_cluster.sql +++ b/kernel/kernel-impl/src/main/sql/mysql/sakai_cluster.sql @@ -4,10 +4,11 @@ CREATE TABLE SAKAI_CLUSTER ( - SERVER_ID VARCHAR (64), + SERVER_ID_INSTANCE VARCHAR (64), UPDATE_TIME TIMESTAMP, - STATUS VARCHAR(8) + STATUS ENUM('STARTING', 'RUNNING', 'CLOSING', 'STOPPING'), + SERVER_ID VARCHAR (64) ); ALTER TABLE SAKAI_CLUSTER - ADD ( PRIMARY KEY (SERVER_ID) ) ; + ADD ( PRIMARY KEY (SERVER_ID_INSTANCE) ) ; diff --git a/kernel/kernel-impl/src/main/sql/oracle/sakai_cluster.sql b/kernel/kernel-impl/src/main/sql/oracle/sakai_cluster.sql index 7a914835994e..8ec3da9a0331 100644 --- a/kernel/kernel-impl/src/main/sql/oracle/sakai_cluster.sql +++ b/kernel/kernel-impl/src/main/sql/oracle/sakai_cluster.sql @@ -4,8 +4,10 @@ CREATE TABLE SAKAI_CLUSTER ( - SERVER_ID VARCHAR2 (64), - UPDATE_TIME TIMESTAMP WITH LOCAL TIME ZONE + SERVER_ID_INSTANCE VARCHAR2 (64), + UPDATE_TIME TIMESTAMP WITH LOCAL TIME ZONE, + STATUS ENUM('STARTING', 'RUNNING', 'CLOSING', 'STOPPING'), + SERVER_ID VARCHAR(64) ); -ALTER TABLE SAKAI_CLUSTER ADD ( CONSTRAINT "SAKAI_CLUSTER_PK" PRIMARY KEY ("SERVER_ID") VALIDATE ); +ALTER TABLE SAKAI_CLUSTER ADD ( CONSTRAINT "SAKAI_CLUSTER_PK" PRIMARY KEY ("SERVER_ID_INSTANCE") VALIDATE ); diff --git a/presence/presence-tool/tool/src/java/org/sakaiproject/presence/tool/PresenceToolAction.java b/presence/presence-tool/tool/src/java/org/sakaiproject/presence/tool/PresenceToolAction.java index 8f7b90c1f4f9..af757a8db537 100644 --- a/presence/presence-tool/tool/src/java/org/sakaiproject/presence/tool/PresenceToolAction.java +++ b/presence/presence-tool/tool/src/java/org/sakaiproject/presence/tool/PresenceToolAction.java @@ -33,6 +33,7 @@ import org.sakaiproject.cheftool.menu.MenuDivider; import org.sakaiproject.cheftool.menu.MenuEntry; import org.sakaiproject.cheftool.menu.MenuImpl; +import org.sakaiproject.cluster.api.ClusterNode; import org.sakaiproject.cluster.api.ClusterService; import org.sakaiproject.component.cover.ComponentManager; import org.sakaiproject.event.api.SessionState; @@ -183,11 +184,17 @@ else if (MODE_SERVERS.equals(state.getAttribute(STATE_DISPLAY_MODE))) Map> session = UsageSessionService.getOpenSessionsByServer(); context.put("serverSessions", session); - Mapstatus = clusterService.getServerStatus(); + Mapnodes = clusterService.getServerStatus(); + // Get a map of statuses + Map status = new HashMap<>(); + for (Map.Entry entry: nodes.entrySet()) { + status.put(entry.getKey(), entry.getValue().getStatus()); + } + context.put("serverStatus", status); Set serverList = new TreeSet(); - serverList.addAll(status.keySet()); + serverList.addAll(nodes.keySet()); serverList.addAll(session.keySet()); context.put("serverList", serverList);