Skip to content

Commit

Permalink
[server] Workaround for SQLite limitation in severity change
Browse files Browse the repository at this point in the history 
In SQLite there is a setting named SQLITE_LIMIT_COMPOUND_SELECT which
limits the maximal number of compound select statements. In changing
checker severities, we have such an SQL statement of which the size is
proportional to the number of checkers. If this limit is exceeded, then
the query fails. The default limit is 500, so the workaround is to
change severities in this big chunks.
  • Loading branch information
@bruntib
bruntib committed Apr 26, 2021
1 parent 169146e commit 47731c5
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 59 deletions.
11 changes: 11 additions & 0 deletions codechecker_common/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""


import itertools
import json
import os

Expand Down Expand Up @@ -148,3 +149,13 @@ def trim_path_prefixes(path, prefixes):
return path

return path[len(longest_matching_prefix):]


def chunks(iterator, n):
"""
Yield the next chunk if an iterable object. A chunk consists of maximum n
elements.
"""
for first in iterator:
rest_of_chunk = itertools.islice(iterator, 0, n - 1)
yield itertools.chain([first], rest_of_chunk)
8 changes: 2 additions & 6 deletions web/server/codechecker_server/api/report_server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1309,9 +1309,7 @@ def getDiffResultsHash(self, run_ids, report_hashes, diff_type,
# For this reason we create queries with chunks.
new_hashes = []
chunk_size = 500
for chunk in [report_hashes[i:i + chunk_size] for
i in range(0, len(report_hashes),
chunk_size)]:
for chunk in util.chunks(iter(report_hashes), chunk_size):
new_hashes_query = union_all(*[
select([bindparam('bug_id' + str(i), h)
.label('bug_id')])
Expand Down Expand Up @@ -2406,9 +2404,7 @@ def __removeReports(self, session, report_ids, chunk_size=500):
"""
Removing reports in chunks.
"""
for r_ids in [report_ids[i:i + chunk_size] for
i in range(0, len(report_ids),
chunk_size)]:
for r_ids in util.chunks(iter(report_ids), chunk_size):
session.query(Report) \
.filter(Report.id.in_(r_ids)) \
.delete(synchronize_session=False)
Expand Down
116 changes: 63 additions & 53 deletions web/server/codechecker_server/database/db_cleanup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from codechecker_api.codeCheckerDBAccess_v6.ttypes import Severity

from codechecker_common import util
from codechecker_common.logger import get_logger

from .database import DBSession
Expand All @@ -24,6 +25,7 @@

LOG = get_logger('server')
RUN_LOCK_TIMEOUT_IN_DATABASE = 30 * 60 # 30 minutes.
SQLITE_LIMIT_COMPOUND_SELECT = 500


def remove_expired_run_locks(session_maker):
Expand Down Expand Up @@ -137,56 +139,64 @@ def upgrade_severity_levels(session_maker, severity_map):
"""
LOG.debug("Upgrading severity levels started...")

with DBSession(session_maker) as session:
try:
# Create a sql query from the severity map.
severity_map_q = union_all(*[
select([cast(bindparam('checker_id' + str(i), str(checker_id))
.label('checker_id'), sqlalchemy.String),
cast(bindparam('severity' + str(i),
Severity._NAMES_TO_VALUES[
severity_map[checker_id]])
.label('severity'), sqlalchemy.Integer)])
for i, checker_id in enumerate(severity_map)]) \
.alias('new_severities')

checker_ids = list(severity_map.keys())

# Get checkers which has been changed.
changed_checker_q = select([Report.checker_id, Report.severity]) \
.group_by(Report.checker_id, Report.severity) \
.where(Report.checker_id.in_(checker_ids)) \
.except_(
session.query(severity_map_q)).alias('changed_severites')

changed_checkers = session.query(changed_checker_q.c.checker_id,
changed_checker_q.c.severity)

# Update severity levels of checkers.
if changed_checkers:
updated_checker_ids = set()
for checker_id, severity_old in changed_checkers:
severity_new = severity_map.get(checker_id, 'UNSPECIFIED')
severity_id = Severity._NAMES_TO_VALUES[severity_new]

LOG.info("Upgrading severity level of '%s' checker from "
"%s to %s",
checker_id,
Severity._VALUES_TO_NAMES[severity_old],
severity_new)

if checker_id in updated_checker_ids:
continue

session.query(Report) \
.filter(Report.checker_id == checker_id) \
.update({Report.severity: severity_id})

updated_checker_ids.add(checker_id)

session.commit()

LOG.debug("Upgrading of severity levels finished...")
except (sqlalchemy.exc.OperationalError,
sqlalchemy.exc.ProgrammingError) as ex:
LOG.error("Failed to upgrade severity levels: %s", str(ex))
for severity_map_small in util.chunks(
iter(severity_map.items()), SQLITE_LIMIT_COMPOUND_SELECT):
severity_map_small = dict(severity_map_small)

with DBSession(session_maker) as session:
try:
# Create a sql query from the severity map.
severity_map_q = union_all(*[
select([cast(bindparam('checker_id' + str(i),
str(checker_id))
.label('checker_id'), sqlalchemy.String),
cast(bindparam('severity' + str(i),
Severity._NAMES_TO_VALUES[
severity_map_small[checker_id]])
.label('severity'), sqlalchemy.Integer)])
for i, checker_id in enumerate(severity_map_small)]) \
.alias('new_severities')

checker_ids = list(severity_map_small.keys())

# Get checkers which has been changed.
changed_checker_q = select(
[Report.checker_id, Report.severity]) \
.group_by(Report.checker_id, Report.severity) \
.where(Report.checker_id.in_(checker_ids)) \
.except_(session.query(severity_map_q)) \
.alias('changed_severites')

changed_checkers = session.query(
changed_checker_q.c.checker_id,
changed_checker_q.c.severity)

# Update severity levels of checkers.
if changed_checkers:
updated_checker_ids = set()
for checker_id, severity_old in changed_checkers:
severity_new = severity_map_small.get(checker_id,
'UNSPECIFIED')
severity_id = Severity._NAMES_TO_VALUES[severity_new]

LOG.info("Upgrading severity level of '%s' checker "
"from %s to %s",
checker_id,
Severity._VALUES_TO_NAMES[severity_old],
severity_new)

if checker_id in updated_checker_ids:
continue

session.query(Report) \
.filter(Report.checker_id == checker_id) \
.update({Report.severity: severity_id})

updated_checker_ids.add(checker_id)

session.commit()

LOG.debug("Upgrading of severity levels finished...")
except (sqlalchemy.exc.OperationalError,
sqlalchemy.exc.ProgrammingError) as ex:
LOG.error("Failed to upgrade severity levels: %s", str(ex))

0 comments on commit 47731c5

Please sign in to comment.