bug 777670 - resurrect old daily_matviews until crontabber is ready

config/crontabber.ini

@@ -3,16 +3,16 @@
     # doc: Note! the hour part (relevant on daily jobs) is in UTC time
     # converter: class_list_converter
     jobs='''socorro.cron.jobs.weekly_reports_partitions.WeeklyReportsPartitionsCronApp|7d
-            socorro.cron.jobs.nightly_builds.NightlyBuildsCronApp|1d|10:00
-            socorro.cron.jobs.matviews.ProductVersionsCronApp|1d|10:00
-            socorro.cron.jobs.matviews.SignaturesCronApp|1d|10:00
-            socorro.cron.jobs.matviews.OSVersionsCronApp|1d|10:00
-            socorro.cron.jobs.matviews.TCBSCronApp|1d|10:00
-            socorro.cron.jobs.matviews.ADUCronApp|1d|10:00
-            socorro.cron.jobs.matviews.DailyCrashesCronApp|1d|10:00
-            socorro.cron.jobs.matviews.HangReportCronApp|1d|10:00
-            socorro.cron.jobs.matviews.RankCompareCronApp|1d|10:00
-            socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d|10:00
+    #       socorro.cron.jobs.nightly_builds.NightlyBuildsCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.ProductVersionsCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.SignaturesCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.OSVersionsCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.TCBSCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.ADUCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.DailyCrashesCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.HangReportCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.RankCompareCronApp|1d|10:00
+    #       socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d|10:00
             socorro.cron.jobs.duplicates.DuplicatesCronApp|1h
             socorro.cron.jobs.reports_clean.ReportsCleanCronApp|1h
             socorro.cron.jobs.bugzilla.BugzillaCronApp|1h'''

puppet/files/etc_crond/socorro

@@ -3,11 +3,6 @@ MAILTO="[email protected]"
 */5 * * * * socorro /data/socorro/application/scripts/crons/crontabber.sh
 # update info for /status page
 */5 * * * * socorro /data/socorro/application/scripts/crons/cron_status.sh
-# run duplicate detection
-02 * * * * socorro /data/socorro/application/scripts/crons/cron_duplicates.sh
-# create "clean" version of incoming reports, only containing valid crashes
-# reportsclean must run after duplicates, see bug 697772
-45 * * * * socorro /data/socorro/application/scripts/crons/cron_reportsclean.sh
 # generate nightly reports for crash-stats UI
 00 02 * * * socorro /data/socorro/application/scripts/crons/cron_daily_matviews.sh
 # pre-create postgresql partitions for larger tables
@@ -16,8 +11,6 @@ MAILTO="[email protected]"
 # NOTE - the rest of the jobs in here are very Mozilla-specific, they
 #        are included for reference but you likely do not need or want them.
 
-## annotate crashes with info from bugzilla
-#11 * * * * socorro /data/socorro/application/scripts/crons/cron_bugzilla.sh
 ## this works around a mozilla-specific breakpad bug
 #22 * * * * socorro /data/socorro/application/scripts/crons/cron_fixbrokendumps.sh
 ## pull product and version info from metadata on an FTP server

scripts/crons/cron_daily_matviews.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+. /etc/socorro/socorrorc
+
+NAME=`basename $0 .sh`
+lock $NAME
+pyjob $NAME startDailyMatviews
+EXIT_CODE=$?
+unlock $NAME
+
+exit $EXIT_CODE

scripts/startDailyMatviews.py

@@ -0,0 +1,43 @@
+#! /usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import sys
+import logging
+import logging.handlers
+from datetime import date, timedelta
+
+try:
+    import config.dailyMatviewsConfig as configModule
+except ImportError:
+    import dailyMatviewsConfig as configModule
+
+import socorro.lib.ConfigurationManager as configurationManager
+import socorro.cron.dailyMatviews as dailyMatviews
+import socorro.lib.util as sutil
+
+try:
+    config = configurationManager.newConfiguration(
+        configurationModule=configModule, applicationName="dailyMatviews 0.1")
+except configurationManager.NotAnOptionError, x:
+    print >>sys.stderr, x
+    print >>sys.stderr, "for usage, try --help"
+    sys.exit()
+
+logger = logging.getLogger("dailyMatviews")
+logger.setLevel(logging.DEBUG)
+
+sutil.setupLoggingHandlers(logger, config)
+sutil.echoConfig(logger, config)
+
+exitCode = 255
+
+try:
+    targetDate = date.today() - timedelta(1)
+    exitCode = dailyMatviews.update(config, targetDate)
+finally:
+    logger.info("done.")
+
+sys.exit(exitCode)

socorro/cron/dailyMatviews.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import sys
+import logging
+import datetime
+import psycopg2
+
+from socorro.lib.datetimeutil import utc_now
+
+logger = logging.getLogger('dailyMatviews')
+logger.addHandler(logging.StreamHandler(sys.stderr))
+
+
+def update(config, targetDate):
+    functions = (
+      # function name, parmeters, dependencies
+      ('update_product_versions', [], []),
+      ('update_signatures', [targetDate], []),
+      ('update_os_versions', [targetDate], []),
+      ('update_tcbs', [targetDate],
+       ['update_product_versions', 'update_signatures', 'update_os_versions']),
+      ('update_adu', [targetDate], []),
+      ('update_daily_crashes', [targetDate],
+       ['update_product_versions' 'update_signatures']),
+      ('update_hang_report', [targetDate], []),
+      ('update_rank_compare', [targetDate], []),
+      ('update_nightly_builds', [targetDate], []),
+    )
+
+    failed = set()
+    databaseDSN = ""
+    if 'databaseHost' in config:
+        databaseDSN += 'host=%(databaseHost)s '
+    if 'databaseName' in config:
+        databaseDSN += 'dbname=%(databaseName)s '
+    if 'databaseUserName' in config:
+        databaseDSN += 'user=%(databaseUserName)s '
+    if 'databasePassword' in config:
+        databaseDSN += 'password=%(databasePassword)s'
+    dsn = databaseDSN % config
+    connection = psycopg2.connect(dsn)
+    cursor = connection.cursor()
+    for funcname, parameters, deps in functions:
+        if set(deps) & failed:
+            # one of the deps previously failed, so skip this one
+            logger.warn("For %r, dependency %s failed so skipping"
+                         % (funcname, ', '.join(set(deps) & failed)))
+            continue
+        logger.info('Running %s' % funcname)
+        failureMessage = None
+        success = False
+        try:
+            cursor.callproc(funcname, parameters)
+            # fetchone() returns a tuple of length 1
+            result = cursor.fetchone()
+            if result and result[0]:
+                success = True
+            else:
+                # "expected" error
+                logger.warn('%r failed' % funcname)
+                failureMessage = '%s did not return true' % funcname
+        except psycopg2.InternalError:
+            # unexpected error
+            logger.error('%r failed' % funcname, exc_info=True)
+            import sys  # don't assume that this has been imported
+            __, error_value = sys.exc_info()[:2]
+            failureMessage = str(error_value)
+        if success:
+            connection.commit()
+        else:
+            connection.rollback()
+            failed.add(funcname)
+
+    return len(failed)

socorro/unittest/cron/testDailyMatviews.py

@@ -0,0 +1,104 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import unittest
+import psycopg2
+import socorro.cron.dailyMatviews  # needed for Mock
+import socorro.cron.dailyMatviews as dailyMatviews
+
+from socorro.lib.datetimeutil import utc_now
+
+from socorro.unittest.config.commonconfig import databaseHost
+from socorro.unittest.config.commonconfig import databaseName
+from socorro.unittest.config.commonconfig import databaseUserName
+from socorro.unittest.config.commonconfig import databasePassword
+
+from mock import patch
+
+
+class mock_connection:
+    def __init__(self, c):
+        self.c = c
+
+    def _void(self):
+        pass
+
+    commit = rollback = _void
+
+    def cursor(self):
+        return self.c
+
+
+class mock_psycopg2:
+    InternalError = Exception
+
+    def __init__(self, cursor):
+        self.cursor = cursor
+
+    def connect(self, *args, **kwargs):
+        return mock_connection(self.cursor)
+
+
+class mock_cursor:
+    def __init__(self, returns):
+        self.returns = returns
+        self.called = []
+
+    def callproc(self, name, params):
+        self.name = name
+        self.called.append(name)
+
+    def fetchone(self):
+        if self.name in self.returns:
+            return self.returns[self.name]
+        return (True,)
+
+    def execute(self, sql, params):
+        pass
+
+
+class TestCase(unittest.TestCase):
+    def setUp(self):
+        self.config = {
+          'databaseHost': '',
+          'databasePassword': '',
+          'databaseName': '',
+          'databaseUserName': '',
+        }
+
+    def test_failing__update_product_versions(self):
+        cursor = mock_cursor({
+          'update_product_versions': (False,),
+        })
+        dailyMatviews.psycopg2 = mock_psycopg2(cursor)
+        with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
+            dailyMatviews.update(self.config, 'some date')
+            self.assertEqual(cursor.called, ['update_product_versions',
+              'update_signatures', 'update_os_versions', 'update_adu',
+              'update_daily_crashes', 'update_hang_report',
+              'update_rank_compare', 'update_nightly_builds'])
+            self.assertEqual(mock_logger.info.call_count, 8)
+            self.assertEqual(mock_logger.warn.call_count, 2)
+            self.assertEqual(mock_logger.error.call_count, 0)
+
+    def test_all_works(self):
+        cursor = mock_cursor({})
+        dailyMatviews.psycopg2 = mock_psycopg2(cursor)
+        with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
+            dailyMatviews.update(self.config, 'some date')
+            self.assertEqual(mock_logger.info.call_count, 9)
+            self.assertEqual(mock_logger.warn.call_count, 0)
+            self.assertEqual(mock_logger.error.call_count, 0)
+
+    def test_mock_internal_error(self):
+        cursor = mock_cursor({
+          'update_signatures': psycopg2.InternalError,
+        })
+        dailyMatviews.psycopg2 = mock_psycopg2(cursor)
+        with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
+            dailyMatviews.update(self.config, 'some date')
+            self.assertEqual(mock_logger.info.call_count, 8)
+            self.assertEqual(mock_logger.warn.call_count, 1)
+            self.assertEqual(mock_logger.error.call_count, 1)