Allow send notifications for all harvest jobs

Co-authored-by: Julie Kramer <[email protected]>

README.rst

@@ -191,7 +191,7 @@ If you want to send an email when a **Harvest Job fails**, you can set the follo
 
     ckan.harvest.status_mail.errored = True
 
-If you want to send an email when **all Harvest Job finishes** (whether or not it failed), you can set the following configuration option in the ini file:
+If you want to send an email when **all Harvest Jobs finish** (whether or not it failed), you can set the following configuration option in the ini file:
 
     ckan.harvest.status_mail.all = True
 

ckanext/harvest/logic/action/update.py

@@ -9,6 +9,7 @@
 
 from ckantoolkit import config
 from sqlalchemy import and_, or_
+from six.moves.urllib.parse import urljoin
 
 from ckan.lib.search.index import PackageSearchIndex
 from ckan.plugins import PluginImplementations
@@ -667,16 +668,15 @@ def harvest_jobs_run(context, data_dict):
                     notify_all = toolkit.asbool(config.get('ckan.harvest.status_mail.all'))
                     notify_errors = toolkit.asbool(config.get('ckan.harvest.status_mail.errored'))
                     last_job_errors = status['last_job']['stats'].get('errored', 0)
-
+                    log.info('Notifications: All:{} On error:{} Errors:{}'.format(notify_all, notify_errors, last_job_errors))
+
                     if notify_all:
-                        subject, body = prepare_summary_mail(context, job_obj.source.id, status)
-                        log.info('Sending summary email')
-                        send_mail(context, job_obj.source.id, subject, body)
+                        if last_job_errors > 0:
+                            send_error_email(context, job_obj.source_id, status)
+                        else:
+                            send_summary_email(context, job_obj.source.id, status)
                     elif notify_errors and last_job_errors > 0:
-                        subject, body = prepare_error_mail(context, job_obj.source_id, status)
-                        log.info('Sending error mail')
-                        send_mail(context, job_obj.source.id, subject, body)
-
+                        send_error_email(context, job_obj.source.id, status)
                 else:
                     log.debug('Ongoing job:%s source:%s',
                               job['id'], job['source_id'])
@@ -693,16 +693,7 @@ def harvest_jobs_run(context, data_dict):
 
 def get_mail_extra_vars(context, source_id, status):
     last_job = status['last_job']
-    harvest_objects = get_action(
-        'harvest_object_list')(context, {'id': source_id})
-    packages = []
 
-    for harvest_object in harvest_objects:
-        object_info = get_action(
-            'harvest_object_show')(context, {'id': harvest_object})
-
-        packages.append(object_info.get('package'))
-
     source = get_action('harvest_source_show')(context, {'id': source_id})
     report = get_action(
         'harvest_job_report')(context, {'id': status['last_job']['id']})
@@ -731,14 +722,14 @@ def get_mail_extra_vars(context, source_id, status):
 
     errors = job_errors + obj_errors
 
+    site_url = config.get('ckan.site_url')
+    job_url = toolkit.url_for('harvest_job_show', source=source['id'], id=last_job['id'])
+    full_job_url = urljoin(site_url, job_url)
     extra_vars = {
         'organization': organization,
         'site_title': config.get('ckan.site_title'),
-        'site_url': config.get('ckan.site_url'),
-        'job_url': toolkit.url_for(
-            'harvest_job_show',
-            source=source['id'],
-            id=last_job['id']),
+        'site_url': site_url,
+        'job_url': full_job_url,
         'harvest_source_title': source['title'],
         'harvest_configuration': harvest_configuration,
         'job_finished': last_job['finished'],
@@ -748,7 +739,6 @@ def get_mail_extra_vars(context, source_id, status):
         'records_added': str(last_job['stats'].get('added', 0)),
         'records_deleted': str(last_job['stats'].get('deleted', 0)),
         'records_updated': str(last_job['stats'].get('updated', 0)),
-        'packages': packages,
         'error_summary_title': toolkit._('Error Summary'),
         'obj_errors_title': toolkit._('Document Error'),
         'job_errors_title': toolkit._('Job Errors'),
@@ -759,21 +749,14 @@ def get_mail_extra_vars(context, source_id, status):
 
     return extra_vars
 
-
 def prepare_summary_mail(context, source_id, status):
     extra_vars = get_mail_extra_vars(context, source_id, status)
     body = render_jinja2('emails/summary_email.txt', extra_vars)
-
-    if str(status['last_job']['stats'].get('errored', 0)) == '0':
-        subject = '{} - Harvesting Job Successful - Summary Notification'\
+    subject = '{} - Harvesting Job Successful - Summary Notification'\
                   .format(config.get('ckan.site_title'))
-    else:
-        subject = '{} - Harvesting Job with Errors - Summary Notification'\
-                  .format(config.get('ckan.site_title'))
-
+
     return subject, body
 
-
 def prepare_error_mail(context, source_id, status):
     extra_vars = get_mail_extra_vars(context, source_id, status)
     body = render_jinja2('emails/error_email.txt', extra_vars)
@@ -782,6 +765,13 @@ def prepare_error_mail(context, source_id, status):
 
     return subject, body
 
+def send_summary_email(context, source_id, status):
+    subject, body = prepare_summary_mail(context, source_id, status)
+    send_mail(context, source_id, subject, body)
+
+def send_error_email(context, source_id, status):
+    subject, body = prepare_error_mail(context, source_id, status)
+    send_mail(context, source_id, subject, body)
 
 def send_mail(context, source_id, subject, body):
     source = get_action('harvest_source_show')(context, {'id': source_id})

ckanext/harvest/templates/emails/error_email.txt

@@ -1,8 +1,8 @@
-This is a failure notification of the latest harvest job ({{ job_url }}) set-up in {{ site_url }}.
+This is a failure notification of the latest harvest job set-up in {{ site_url }}. 
+Job URL: {{ job_url }}
 
 Harvest Source: {{ harvest_source_title }}
 Harvest Configuration: {{ harvest_configuration | safe }}
-
 Organization: {{ organization }}
 
 Harvest Job Id: {{ job_id }}
@@ -14,20 +14,14 @@ Records Added: {{ records_added }}
 Records Updated: {{ records_updated }}
 Records Deleted: {{ records_deleted }}
 
-{{ error_summary_title }}
- - {{ errors|length }} errors
+{{ error_summary_title }}: {{ errors|length }} errors
+
 {{ job_errors_title }}: {{ job_errors|length }}
 {% for error in job_errors %}
     - {{ error }} {% endfor %}
-
 {{ obj_errors_title }}: {{ obj_errors|length }}
 {% for error in obj_errors %}
     - {{ error }} {% endfor %}
-
-Total packages: {{ packages|length }}
-{% for package in packages %}
-    - {{ package }}{% endfor %}
-
 --
 You are receiving this email because you are currently set-up as Administrator for {{ site_url }}.
 Please do not reply to this email as it was sent from a non-monitored address.

ckanext/harvest/templates/emails/summary_email.txt

@@ -1,4 +1,5 @@
-This is a summary of the latest harvest job ({{ job_url }}) set-up in {{ site_url }}.
+This is a summary of the latest harvest job set-up in {{ site_url }}.
+Job URL: {{ job_url }}
 
 Harvest Source: {{ harvest_source_title }}
 Harvest Configuration: {{ harvest_configuration | safe }}
@@ -14,8 +15,7 @@ Records Added: {{ records_added }}
 Records Updated: {{ records_updated }}
 Records Deleted: {{ records_deleted }}
 
-{{ error_summary_title }}
- - {{ errors|length }} errors
+{{ error_summary_title }}: {{ errors|length }} errors
 {{ job_errors_title }}: {{ job_errors|length }}
 {% for error in job_errors %}
     - {{ error }} {% endfor %}
@@ -24,10 +24,6 @@ Records Deleted: {{ records_deleted }}
 {% for error in obj_errors %}
     - {{ error }} {% endfor %}
 
-Total packages: {{ packages|length }}
-{% for package in packages %}
-    - {{ package }}{% endfor %}
-
 --
 You are receiving this email because you are currently set-up as Administrator for {{ site_url }}.
 Please do not reply to this email as it was sent from a non-monitored address.