SAK-39934 - Improvements wrt unpublishing sites with the Course Site …

…Removal job (sakaiproject#5545)

config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties

@@ -4892,12 +4892,20 @@
 # the actions can be: remove or unpublish (by default)
 #course_site_removal_service.action=unpublish
 
+# When true, this bypasses the standard hooks of saving sites; this significantly boosts the job's performance
+# Set this to true only in conjuction with course_site_removal_service.action=unpublish
+#course_site_removal_service.silently.unpublish=false
+
 #The user that will execute the removal/unpublish job
 #course_site_removal_service.user=admin
 
 # The grace period after the term ends
 #course_site_removal_service.num_days_after_term_ends=14
 
+# It is possible for a course to be attached to multiple sections that fall in different terms
+# When this is true, a given site will only be unpublished if all of its associated sections' terms' end dates have elapsed before the grace period
+#course_site_removal_service.handle.crosslisting=false
+
 
 # Tool ID to use when adding users to a site.
 # Can also be overridden by adding to site properties.

edu-services/cm-service/cm-impl/hibernate-pack/src/webapp/WEB-INF/components.xml

@@ -212,11 +212,13 @@
 		  class="org.sakaiproject.coursemanagement.impl.CourseSiteRemovalServiceImpl"
 		  destroy-method="destroy"
 		  init-method="init">
-		<property name="courseManagementService"><ref bean="org.sakaiproject.coursemanagement.api.CourseManagementService"      /></property>   <!-- used to get the start and end date of terms       -->
-		<property name="functionManager"        ><ref bean="org.sakaiproject.authz.api.FunctionManager"                         /></property>   <!-- used to register api permissions                  -->
-		<property name="securityService"        ><ref bean="org.sakaiproject.authz.api.SecurityService"                         /></property>   <!-- used to check for super user                      -->
-		<property name="sessionFactory"         ><ref bean="org.sakaiproject.springframework.orm.hibernate.GlobalSessionFactory"/></property>   <!-- global hibernate transaction manager              -->
-		<property name="siteService"            ><ref bean="org.sakaiproject.site.api.SiteService"                              /></property>   <!-- needed to retrieve course sites that have expired -->
+		<property name="authzGroupService"         ><ref bean="org.sakaiproject.authz.api.AuthzGroupService"                       /></property>   <!-- used to get providers from a site                 -->
+		<property name="courseManagementService"   ><ref bean="org.sakaiproject.coursemanagement.api.CourseManagementService"      /></property>   <!-- used to get the start and end date of terms       -->
+		<property name="functionManager"           ><ref bean="org.sakaiproject.authz.api.FunctionManager"                         /></property>   <!-- used to register api permissions                  -->
+		<property name="securityService"           ><ref bean="org.sakaiproject.authz.api.SecurityService"                         /></property>   <!-- used to check for super user                      -->
+		<property name="serverConfigurationService"><ref bean="org.sakaiproject.component.api.ServerConfigurationService"          /></property>   <!-- to get sakai.properties                           -->
+		<property name="sessionFactory"            ><ref bean="org.sakaiproject.springframework.orm.hibernate.GlobalSessionFactory"/></property>   <!-- global hibernate transaction manager              -->
+		<property name="siteService"               ><ref bean="org.sakaiproject.site.api.SiteService"                              /></property>   <!-- needed to retrieve course sites that have expired -->
 	</bean>
 
 	<!-- global hibernate transaction proxy bean -->

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/jobs/coursepublish/CourseSiteRemovalJob.java

@@ -104,22 +104,22 @@ public void init() {
          user = userDirectoryService.getUser(userId);
       } catch (UserNotDefinedException ex) {
          user = null;
-         log.error("The user with eid " + userId + " was not found.  The course site publish job has been aborted.");
+         log.error("The user with eid {} was not found.  The course site publish job has been aborted.", userId);
       }
 
 
       // get the action to be taken when a course is found to be expired
       String actionString = serverConfigurationService.getString(PROPERTY_COURSE_SITE_REMOVAL_ACTION);
       if (actionString == null || actionString.trim().length() == 0)
       {
-         log.warn("The property " + PROPERTY_COURSE_SITE_REMOVAL_ACTION + " was not specified in sakai.properties.  Using a default value of " + DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION + ".");
+         log.warn("The property " + PROPERTY_COURSE_SITE_REMOVAL_ACTION + " was not specified in sakai.properties.  Using a default value of {}.", DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION);
          action = DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION;
       }
       else
       {
          action = getAction(actionString);
          if (action == null) {
-            log.error("The value specified for " + actionString + " in sakai.properties, " + PROPERTY_COURSE_SITE_REMOVAL_ACTION + ", is not valid.  A default value of " + DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION + " will be used instead.");
+            log.error("The value specified for {} in sakai.properties, " + PROPERTY_COURSE_SITE_REMOVAL_ACTION + ", is not valid.  A default value of {} will be used instead.", actionString, DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION);
             action = DEFAULT_VALUE_COURSE_SITE_REMOVAL_ACTION;
          }
       }
@@ -132,19 +132,20 @@ public void init() {
    public void execute(JobExecutionContext context) throws JobExecutionException {
       synchronized (this) {
          log.info("execute()");
+         String actionStr = CourseSiteRemovalService.Action.remove.equals(action) ? " course sites were removed." : " course sites were unpublished.";
 
          if (user == null) {
-            log.error("The scheduled job to remove course sites can not be run with an invalid user.  No courses were removed.");
+            log.error("The scheduled job to remove course sites can not be run with an invalid user.  No{}", actionStr);
          } else {
             try {
                // switch the current user to the one specified to run the quartz job
                Session sakaiSesson = sessionManager.getCurrentSession();
                sakaiSesson.setUserId(user.getId());
 
                int numSitesRemoved = courseSiteRemovalService.removeCourseSites(action, numDaysAfterTermEnds);
-               log.info(numSitesRemoved + " course sites were removed.");
+               log.info("{}{}", numSitesRemoved, actionStr);
             } catch (Exception ex) {
-               log.error(ex.getMessage());
+               log.error(ex.getMessage(), ex);
             }
          }
       }

kernel/api/src/main/java/org/sakaiproject/site/api/SiteService.java

@@ -226,26 +226,37 @@ public interface SiteService extends EntityProducer
 	 */
 	public class SelectionType
 	{
+		public enum PublishedFilter {ALL, PUBLISHED_ONLY, UNPUBLISHED_ONLY}
+
 		private final String m_id;
 
 		private final boolean m_ignoreSpecial;
 
 		private final boolean m_ignoreUser;
 
-		private final boolean m_ignoreUnpublished;
+		private final PublishedFilter m_publishedFilter;
 
 		//always true, we always ignore unpublished sites
 		private final boolean m_ignoreSoftlyDeleted;
 
-		private SelectionType(String id, boolean ignoreSpecial, boolean ignoreUser, boolean ignoreUnpublished, boolean ignoreSoftlyDeleted)
+		public SelectionType(String id, boolean ignoreSpecial, boolean ignoreUser, boolean ignoreUnpublished, boolean ignoreSoftlyDeleted)
 		{
 			m_id = id;
 			m_ignoreSpecial = ignoreSpecial;
 			m_ignoreUser = ignoreUser;
-			m_ignoreUnpublished = ignoreUnpublished;
+			m_publishedFilter = ignoreUnpublished ? PublishedFilter.PUBLISHED_ONLY : PublishedFilter.ALL;
 			m_ignoreSoftlyDeleted =  ignoreSoftlyDeleted;
 		}
 
+		public SelectionType(String id, boolean ignoreSpecial, boolean ignoreUser, PublishedFilter publishedFilter, boolean ignoreSoftlyDeleted)
+		{
+			m_id = id;
+			m_ignoreSpecial = ignoreSpecial;
+			m_ignoreUser = ignoreUser;
+			m_publishedFilter = publishedFilter;
+			m_ignoreSoftlyDeleted = ignoreSoftlyDeleted;
+		}
+
 		public String toString()
 		{
 			return m_id;
@@ -263,7 +274,12 @@ public boolean isIgnoreUser()
 
 		public boolean isIgnoreUnpublished()
 		{
-			return m_ignoreUnpublished;
+			return m_publishedFilter == PublishedFilter.PUBLISHED_ONLY;
+		}
+
+		public PublishedFilter getPublishedFilter()
+		{
+			return m_publishedFilter;
 		}
 
 		public boolean isIgnoreSoftlyDeleted()
@@ -1134,6 +1150,33 @@ public boolean isAsc()
 	 */
 	List<String> getSiteIds(SelectionType type, Object ofType, String criteria, Map<String, String> propertyCriteria, SortType sort, PagingPosition page);
 
+	/**
+	 * Get the Site IDs for all sites matching criteria.
+	 * This is useful when you only need the listing of site ids (for other operations) and do not need the actual Site objects.
+	 *
+	 *
+	 * All parameters are the same as {@link #getSites(org.sakaiproject.site.api.SiteService.SelectionType, Object, String, Map, org.sakaiproject.site.api.SiteService.SortType, PagingPosition)}
+	 *
+	 * @param type
+	 *        The SelectionType specifying what sort of selection is intended.
+	 * @param ofType
+	 *        Site type criteria: null for any type; a String to match a single type; A String[], List or Set to match any type in the collection.
+	 * @param criteria
+	 *        Additional selection criteria: sites returned will match this string somewhere in their id, title, description, or skin.
+	 * @param propertyCriteria
+	 *        Additional selection criteria: sites returned will have a property named to match each key in the map, whose values match (somewhere in their value) the value in the map (may be null or empty).
+	 * @param propertyRestrictions
+	 *        Similar to propertyCriteria: returned siteIds will be filtered to exclude any site whose site properties match these key-value pairs
+	 * @param sort
+	 *        A SortType indicating the desired sort. For no sort, set to SortType.NONE.
+	 * @param page
+	 *        The PagePosition subset of items to return.
+	 * @param userId
+	 *        Returned sites will be those which can be accessed by this user
+	 * @return a List of the Site IDs for the sites matching the criteria.
+	 */
+	List<String> getSiteIds(SelectionType type, Object ofType, String criteria, Map<String, String> propertyCriteria, Map<String, String> propertyRestrictions, SortType sort, PagingPosition page, String userId);
+
 	/**
 	 * Get all sites that have been softly deleted
 	 * 
@@ -1300,4 +1343,23 @@ public boolean isAsc()
 	 * @param siteProviders the site providers corresponding to the specified site; if null, they will be looked up
 	 */
 	public String getUserSpecificSiteTitle(Site site, String userID, List<String> siteProviders);
+
+	/**
+	 * Unpublishes the specified sites without touching authz groups and bypasses any SiteAdvisors / other observing services.
+	 * NB:  use this only when making very minimal changes in performance critical tasks.
+	 * Only side effect is events posted to EventTrackingService.
+	 * @param siteIds site IDs for sites to be unpublished;
+	 * siteIds cannot be null
+	 */
+	public void silentlyUnpublish(List<String> siteIds);
+
+	/**
+	 * Saves a site property for the sites with the specified IDs using the specified name-value pair in a single transaction.
+	 * NB: inserts only; doesn't do any duplicate checking. Vulnerable to unique constraint violations
+	 * Use this only when making very minimal changes in performance critical tasks.
+	 * @param propertyName the site property name
+	 * @param propertyValue the site property value
+	 * @param siteIds site Ids on which to insert the specified property name-value pair
+	 */
+	public void saveSitePropertyOnSites(String propertyName, String propertyValue, String... siteIds);
 }

kernel/kernel-impl/src/main/java/org/sakaiproject/site/impl/BaseSiteService.java

@@ -1158,6 +1158,43 @@ public List<String> getSiteTypeStrings(String type)
 		return Arrays.asList(siteTypes);
 	}
 
+	/**
+	 * @inheritDoc
+	 */
+	@Override
+	public void silentlyUnpublish(List<String> siteIds)
+	{
+		if (siteIds == null)
+		{
+			throw new IllegalArgumentException("siteIds cannot be null");
+		}
+
+		String currentUser = sessionManager().getCurrentSessionUserId();
+		Time lastModifiedTime = timeService().newTime();
+
+		// complete the edit
+		storage().unpublish(siteIds, currentUser, lastModifiedTime);
+
+		// track it
+		String event = SECURE_UPDATE_SITE;
+		for (String siteId : siteIds)
+		{
+			String siteReference = siteReference(siteId);
+			eventTrackingService().post(eventTrackingService().newEvent(event, siteReference, true));
+		}
+	}
+
+	/**
+	 * Saves a site property for the sites with the specified IDs using the specified name-value pair in a single transaction.
+	 * NB: inserts only; doesn't do any duplicate checking. Vulnerable to unique constraint violations
+	 * Use this only when making very minimal changes in performance critical tasks.
+	 */
+	@Override
+	public void saveSitePropertyOnSites(String propertyName, String propertyValue, String... siteIds)
+	{
+		storage().writeProperty(propertyName, propertyValue, siteIds);
+	}
+
 	private boolean isCourseSite(String siteId) {
 		boolean rv = false;
 		try {
@@ -2015,6 +2052,14 @@ private List<Site> getUserSitesByPublishedStatus( boolean requireDescription, St
 		}
 	}
 
+	/**
+	 * @inheritDoc
+	 */
+	public List<String> getSiteIds(SelectionType type, Object ofType, String criteria, Map<String, String> propertyCriteria, Map<String, String> propertyRestrictions, SortType sort, PagingPosition page, String userId)
+	{
+		return storage().getSiteIds(type, ofType, criteria, propertyCriteria, propertyRestrictions, null, sort, page, userId);
+	}
+
 	/**
 	 * @inheritDoc
 	 */
@@ -2939,6 +2984,29 @@ protected interface Storage
 		 */
 		public void saveInfo(String siteId, String description, String infoUrl);
 
+		/**
+		 * Unpublish the sites by simply unsetting the PUBLISHED flag
+		 * @param siteIds
+		 *        The site to unpublish
+		 * @param modifedBy
+		 *        User who is unpublishing the site (as a userID)
+		 * @param modifiedOn
+		 *        Time that the site is unpublished
+		 */
+		public void unpublish(List<String> siteIds, String modifiedBy, Time modifiedOn);
+
+		/**
+		 * Writes site properties
+		 */
+		public void writeProperties(Entity r, ResourceProperties props);
+
+		/**
+		 * Saves a site property for the sites with the specified IDs using the specified name-value pair in a single transaction.
+		 * NB: inserts only; doesn't do any duplicate checking. Vulnerable to unique constraint violations
+		 * Use this only when making very minimal changes in performance critical tasks.
+		 */
+		public void writeProperty(String propertyName, String propertyValue, String... siteId);
+
 		/**
 		 * Remove this site.
 		 * 
@@ -3129,6 +3197,34 @@ public List<Site> getSites(SelectionType type, Object ofType, String criteria, M
 		 */
 		List<String> getSiteIds(SelectionType type, Object ofType, String criteria, Map<String, String> propertyCriteria, SortType sort, PagingPosition page);
 
+		/**
+		 * Get the Site IDs for all sites matching criteria.
+		 * This is useful when you only need the listing of site ids (for other operations) and do not need the actual Site objects.
+		 *
+		 * All parameters are the same as {@link #getSites(org.sakaiproject.site.api.SiteService.SelectionType, Object, String, Map, org.sakaiproject.site.api.SiteService.SortType, PagingPosition)}
+		 *
+		 * @param type
+		 *        The SelectionType specifying what sort of selection is intended.
+		 * @param ofType
+		 *        Site type criteria: null for any type; a String to match a single type; A String[], List or Set to match any type in the collection.
+		 * @param criteria
+		 *        Additional selection criteria: sites returned will match this string somewhere in their id, title, description, or skin.
+		 * @param propertyCriteria
+		 *        Additional selection criteria: sites returned will have a property named to match each key in the map, whose values match (somewhere in their value) the value in the map (may be null or empty).
+		 * @param propertyRestrictions
+		 *        Similar to propertyCriteria, except matches will be excluded
+		 * @param excludedSites
+		 *        siteIds to be excluded from the results
+		 * @param sort
+		 *        A SortType indicating the desired sort. For no sort, set to SortType.NONE.
+		 * @param page
+		 *        The PagePosition subset of items to return.
+		 * @param userId
+		 *        The returned sites will be those which can be accessed by the user with this internal ID
+		 * @return a List of the Site IDs for the sites matching the criteria.
+		 */
+		List<String> getSiteIds(SelectionType type, Object ofType, String criteria, Map<String, String> propertyCriteria, Map<String, String> propertyRestrictions, List<String> excludedSites, SortType sort, PagingPosition page, String userId);
+
 		/**
 		 * Count the Site objets that meet specified criteria.
 		 *