Added reminder functionality

.github/workflows/flake8.yml

@@ -19,4 +19,7 @@ jobs:
       - name: Lint with flake8
         run: |
           pip install flake8
-          flake8 --ignore=W191,E117,E501,E722 main.py
+          flake8 --ignore=E117,E501,E722,W191 jeeves.py
+          flake8 --ignore=E117,E501,E722,W191 report.py
+          flake8 --ignore=E117,E501,E722,W191 remind.py
+          flake8 --ignore=E117,E501,E722,W191 functions.py

README.md

@@ -30,12 +30,20 @@ If you have a blocker for a job that is neither a Bugzilla bug or a Jira ticket,
 
 If you wish to use a different blockers file, you can specify it as a command line argument.
 
+#### Tracking Owners
+You can define "owners" for a job in "blockers.yaml" for use with reminder mode. To do so, simply add an "owners" subfield to a job with one or more emails. You can see some examples of this in "blockers.yaml.example". 
+
 ## Usage
 To run:
-- `$ ./main.py [optional: --config CONFIG] [optional: --blockers BLOCKERS]` if `/usr/bin/python3` is a valid path
-- `$ python3 main.py [optional: --config CONFIG] [optional: --blockers BLOCKERS]` otherwise
+- `$ ./jeeves.py [optional: --config CONFIG] [optional: --blockers BLOCKERS]` if `/usr/bin/python3` is a valid path
+- `$ python3 jeeves.py [optional: --config CONFIG] [optional: --blockers BLOCKERS]` otherwise
 - To send report to email specified in `email_to_test` field, add `--test`
 - To save report to 'archive' folder, add `--save`
+- To run Jeeves in "reminder" mode, add `--remind`
+    - Note this will override the usage of `--save` and `--test`
+
+#### Reminder Mode
+Jeeves has a reminder mode that will send an email to "owners" of jobs in Jenkins that have "UNSTABLE" or "FAILURE" status and have no recorded blockers.
 
 ### Packages
 - [PyYAML](https://pyyaml.org/) for parsing config YAML
@@ -49,8 +57,7 @@ To install packages run:
 `$ pip install -r requirements.txt`
 
 ## Testing
-
-Jeeves has a small but growing test suite driven by [pytest](https://docs.pytest.org/en/latest/index.html). Currently all tests reside in the `test_main.py` file.
+Jeeves has a small but growing test suite driven by [pytest](https://docs.pytest.org/en/latest/index.html). Currently all tests reside in the `test_functions.py` file.
 
 To run tests simply run the `pytest` command within the Jeeves directory.
 

blockers.yaml.example

@@ -5,25 +5,27 @@ job1:
     jira:
         - 'RHOSINFRA-123'
 
-# 0 indicates blocker bug/ticket is not on file (either doesn't exist or hasn't been created yet)
 job2:
-    bz:
-        - 0
-    jira:
-        - 0
-
-job3:
     bz:
         - 456
         - 789
     jira:
         - 'RHOSINFRA-456'
         - 'RHOSINFRA-789'
 
-# Jobs with blockers that are neither blockers nor tickets can utilize the 'other' field for recording
+# 0 indicates blocker bug/ticket is not on file (either doesn't exist or hasn't been created yet)
+job3:
+    bz:
+        - 0
+    jira:
+        - 0
+
+# Jobs with blockers that are neither bugs nor tickets can utilize the 'other' field for recording
 job4:
     bz:
         - 159
+    jira:
+        - 0
     other:
         - name: Trello Card
           url: <URL to Trello card>
@@ -32,8 +34,29 @@ job4:
 
 # You can have 'other' blockers with a name, a URL, or both
 job5:
+    bz:
+        - 0
     jira:
         - 'RHOSINFRA-753'
     other:
         - name: Experimental Job
         - url: <URL>
+
+# If you wish for a job to have an owner, you can specify this as follows
+job6:
+    owners:
+        - [email protected]
+    bz:
+        - 0
+    jira:
+        - 'RHOSINFRA-963'
+
+# Jobs can have multiple owners
+job7:
+    owners:
+        - [email protected]
+        - [email protected]
+    bz:
+        - 852
+    jira:
+        - 0

functions.py

@@ -0,0 +1,189 @@
+''' Shared library of functions for report.py and remind.py
+'''
+
+import os
+import re
+import datetime
+import bugzilla
+from jira import JIRA
+
+
+def generate_html_file(htmlcode, remind=False):
+	''' generates HTML file of reminder
+	'''
+	try:
+		os.makedirs('archive')
+	except FileExistsError:
+		pass
+	reportType = 'reminder' if remind else 'report'
+	filename = './archive/{}_{:%m%d%Y_%H:%M:%S}.html'.format(
+		reportType, datetime.datetime.now())
+	with open(filename, 'w') as file:
+		file.write(htmlcode)
+	return None
+
+
+def get_bugs_dict(bug_ids, config):
+	''' takes in set of bug_ids and returns dictionary with
+		bug_ids as keys and API data as values
+	'''
+
+	# initialize bug dictionary
+	bugs = {}
+
+	# iterate through bug ids from set
+	for bug_id in bug_ids:
+
+		# 0 should be default in YAML file (i.e. no bugs recorded)
+		# if present reference should be made in bug dict
+		if bug_id == 0:
+			bugs[0] = {'bug_name': 'No bug on file', 'bug_url': None}
+			continue
+
+		# get bug info from bugzilla API
+		try:
+
+			# hotfix: API call does not work if '/' present at end of URL string
+			parsed_bz_url = config['bz_url'].rstrip('/')
+
+			bz_api = bugzilla.Bugzilla(parsed_bz_url)
+			bug = bz_api.getbug(bug_id)
+			bug_name = bug.summary
+		except Exception as e:
+			print("Bugzilla API Call Error: ", e)
+			bug_name = "BZ#" + str(bug_id)
+		finally:
+			bug_url = config['bz_url'] + "/show_bug.cgi?id=" + str(bug_id)
+			bugs[bug_id] = {'bug_name': bug_name, 'bug_url': bug_url}
+
+	return bugs
+
+
+def get_bugs_set(blockers):
+	''' takes in blockers object and generates a set of all unique bug ids
+		including 0 if it is present
+	'''
+	bug_set = set()
+	for job in blockers:
+		bz = blockers[job]['bz']
+		bug_set.update(bz)
+	return bug_set
+
+
+def get_jenkins_jobs(server, job_search_fields):
+	''' takes in a Jenkins server object and job_search_fields string
+		returns list of jobs with given search field as part of their name
+	'''
+
+	# parse list of search fields
+	fields = job_search_fields.split(',')
+
+	# fetch all jobs from server
+	all_jobs = server.get_jobs()
+
+	# parse out all jobs that do not contain any search field and/or are not OSP10, OSP13, OSP15 or OSP16 jobs
+	relevant_jobs = []
+	supported_versions = ['10', '13', '15', '16', '16.1']
+	for job in all_jobs:
+		job_name = job['name']
+		if any(supported_version in job_name for supported_version in supported_versions):
+			for field in fields:
+				if field in job_name:
+					relevant_jobs.append(job)
+					break
+
+	return relevant_jobs
+
+
+def get_jira_dict(ticket_ids, config):
+	''' takes in set of ticket_ids and returns dictionary with
+		ticket_ids as keys and API data as values
+	'''
+
+	# initialize ticket dictionary
+	tickets = {}
+
+	# initialize connection
+	auth = (config['jira_username'], config['jira_password'])
+	options = {
+		"server": config['jira_url'],
+		"verify": config['certificate']
+	}
+	jira = JIRA(auth=auth, options=options)
+
+	# iterate through ticket ids from set
+	for ticket_id in ticket_ids:
+
+		# 0 should be default in YAML file (i.e. no tickers recorded)
+		# if there is a 0 entry then that should be the only "ticket", so break
+		if ticket_id == 0:
+			tickets[0] = {'ticket_name': 'No ticket on file', 'ticket_url': None}
+			continue
+
+		# get ticket info from jira API
+		try:
+			issue = jira.issue(ticket_id)
+			ticket_name = issue.fields.summary
+		except Exception as e:
+			print("Jira API Call Error: ", e)
+			ticket_name = ticket_id
+		finally:
+			ticket_url = config['jira_url'] + "/browse/" + str(ticket_id)
+			tickets[ticket_id] = {
+				'ticket_name': ticket_name,
+				'ticket_url': ticket_url
+			}
+	jira.close()
+
+	return tickets
+
+
+def get_jira_set(blockers):
+	''' takes in blockers object and generates a set of all unique jira ticket ids
+		including 0 if it is present
+	'''
+	jira_set = set()
+	for job in blockers:
+		jira = blockers[job]['jira']
+		jira_set.update(jira)
+	return jira_set
+
+
+def get_osp_version(job_name):
+	''' gets osp version from job name via regex
+		returns None if no version is found
+	'''
+	version = re.search(r'\d+\.*\d*', job_name)
+	if version is None:
+		return None
+	return version.group()
+
+
+def get_other_blockers(blockers, job_name):
+	''' takes in blockers object and job name
+		returns list of 'other' blockers
+	'''
+
+	other_blockers = blockers[job_name]['other']
+	other = []
+	for blocker in other_blockers:
+		other.append({'other_name': blocker.get('name', 'Link'), 'other_url': blocker.get('url', None)})
+	return other
+
+
+def has_blockers(blockers, job_name):
+	''' returns True if job_name in blockers has any defined blockers
+		returns False otherwise
+	'''
+	is_bz = blockers[job_name].get('bz', [0])
+	is_jira = blockers[job_name].get('jira', [0])
+	is_other = blockers[job_name].get('other', [0])
+	if (is_bz == [0]) and (is_jira == [0]) and (is_other == [0]):
+		return False
+	return True
+
+
+def percent(part, whole):
+	''' basic percent function
+	'''
+	return round(100 * float(part) / float(whole), 1)

jeeves.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import yaml
+import jenkins
+import argparse
+import datetime
+
+from report import run_report
+from remind import run_remind
+
+os.environ['PYTHONHTTPSVERIFY'] = '0'
+
+
+def generate_report_header(user, job_search_fields):
+	''' generates header for report
+	'''
+	user_properties = user['property']
+	user_email_address = [prop['address'] for prop in user_properties if prop['_class'] == 'hudson.tasks.Mailer$UserProperty'][0]
+	date = '{:%m/%d/%Y at %I:%M%p %Z}'.format(datetime.datetime.now())
+	header = {
+		'user_email_address': user_email_address,
+		'date': date,
+		'job_search_fields': job_search_fields
+	}
+	return header
+
+
+def generate_remind_header(user, blocker_file):
+	''' generates header for reminder
+	'''
+	user_properties = user['property']
+	user_email_address = [prop['address'] for prop in user_properties if prop['_class'] == 'hudson.tasks.Mailer$UserProperty'][0]
+	date = '{:%m/%d/%Y at %I:%M%p %Z}'.format(datetime.datetime.now())
+	header = {
+		'user_email_address': user_email_address,
+		'date': date,
+		'blocker_file': blocker_file
+	}
+	return header
+
+
+# main script execution
+if __name__ == '__main__':
+
+	# argument parsing
+	parser = argparse.ArgumentParser(description='An automated report generator for Jenkins CI')
+	parser.add_argument("--config", default="config.yaml", type=str, help='Configuration YAML file to use')
+	parser.add_argument("--blockers", default="blockers.yaml", type=str, help='Blockers YAML file to use')
+	parser.add_argument("--test", default=False, action='store_true', help='Flag to send email to test address')
+	parser.add_argument("--save", default=False, action='store_true', help='Flag to save report to archives')
+	parser.add_argument("--remind", default=False, action='store_true', help='Flag to run Jeeves in "reminder" mode. Note this will override --test and --save')
+	args = parser.parse_args()
+	config_file = args.config
+	blocker_file = args.blockers
+	test = args.test
+	save = args.save
+	remind_flag = args.remind
+
+	# load configuration data
+	try:
+		with open(config_file, 'r') as file:
+			config = yaml.safe_load(file)
+	except Exception as e:
+		print("Error loading configuration data: ", e)
+		sys.exit()
+
+	# load blocker data
+	try:
+		with open(blocker_file, 'r') as file:
+			blockers = yaml.safe_load(file)
+	except Exception as e:
+		print("Error loading blocker configuration data: ", e)
+		sys.exit()
+
+	# connect to jenkins server
+	try:
+		server = jenkins.Jenkins(config['jenkins_url'], username=config['jenkins_username'], password=config['jenkins_api_token'])
+		user = server.get_whoami()
+	except Exception as e:
+		print("Error connecting to Jenkins server: ", e)
+		sys.exit()
+
+	# execute Jeeves in either 'remind' or 'report' mode
+	if remind_flag:
+		header = generate_remind_header(user, blocker_file)
+		run_remind(config, blockers, server, header)
+	else:
+		header = generate_report_header(user, config['job_search_fields'])
+		run_report(config, blockers, server, header, test, save)