Add history truncation to maintenance task

When performing background maintenance, also truncate history
tables, currently dropping entries older than one year.  Fix a bug
in how the command-line maintenance invocation was being tested.

src/gafaelfawr/cli.py

@@ -192,14 +192,16 @@ async def maintenance(settings: Optional[str]) -> None:
         config_dependency.set_settings_path(settings)
     config = await config_dependency()
     logger = structlog.get_logger("gafaelfawr")
-    logger.debug("Starting")
+    logger.debug("Starting background maintenance")
     engine = create_database_engine(
         config.database_url, config.database_password
     )
     async with Factory.standalone(config, engine, check_db=True) as factory:
         token_service = factory.create_token_service()
         async with factory.session.begin():
             await token_service.expire_tokens()
+            await token_service.truncate_history()
+    logger.debug("Completed background maintenance")
 
 
 @main.command()

src/gafaelfawr/constants.py

@@ -1,5 +1,33 @@
 """Constants for Gafaelfawr."""
 
+from datetime import timedelta
+
+__all__ = [
+    "ACTOR_REGEX",
+    "ALGORITHM",
+    "BOT_USERNAME_REGEX",
+    "CHANGE_HISTORY_RETENTION",
+    "COOKIE_NAME",
+    "CURSOR_REGEX",
+    "GID_MIN",
+    "GID_MAX",
+    "GROUPNAME_REGEX",
+    "HTTP_TIMEOUT",
+    "ID_CACHE_SIZE",
+    "LDAP_CACHE_SIZE",
+    "LDAP_CACHE_LIFETIME",
+    "LDAP_TIMEOUT",
+    "MINIMUM_LIFETIME",
+    "OIDC_AUTHORIZATION_LIFETIME",
+    "SETTINGS_PATH",
+    "SCOPE_REGEX",
+    "TOKEN_CACHE_SIZE",
+    "UID_BOT_MIN",
+    "UID_BOT_MAX",
+    "UID_USER_MIN",
+    "USERNAME_REGEX",
+]
+
 ALGORITHM = "RS256"
 """JWT algorithm to use for all tokens."""
 
@@ -12,6 +40,9 @@
 LDAP_TIMEOUT = 5.0
 """Timeout (in seconds) for LDAP queries."""
 
+CHANGE_HISTORY_RETENTION = timedelta(days=365)
+"""Retention of old token change history entries."""
+
 MINIMUM_LIFETIME = 5 * 60
 """Minimum expiration lifetime for a token in seconds."""
 

src/gafaelfawr/services/token.py

@@ -11,7 +11,11 @@
 from structlog.stdlib import BoundLogger
 
 from ..config import Config
-from ..constants import MINIMUM_LIFETIME, USERNAME_REGEX
+from ..constants import (
+    CHANGE_HISTORY_RETENTION,
+    MINIMUM_LIFETIME,
+    USERNAME_REGEX,
+)
 from ..exceptions import (
     InvalidExpiresError,
     InvalidIPAddressError,
@@ -454,7 +458,7 @@ async def get_change_history(
 
         Returns
         -------
-        entries : List[`gafaelfawr.models.history.TokenChangeHistoryEntry`]
+        entries : `gafaelfawr.models.history.PaginatedHistory`
             A list of changes matching the search criteria.
 
         Raises
@@ -782,6 +786,11 @@ async def modify_token(
         )
         return info
 
+    async def truncate_history(self) -> None:
+        """Drop history entries older than the cutoff date."""
+        cutoff = current_datetime() - CHANGE_HISTORY_RETENTION
+        await self._token_change_store.delete(older_than=cutoff)
+
     def _check_authorization(
         self,
         username: Optional[str],

src/gafaelfawr/storage/history.py

@@ -6,7 +6,7 @@
 from typing import Optional
 
 from safir.database import datetime_from_db, datetime_to_db
-from sqlalchemy import and_, func, or_
+from sqlalchemy import and_, delete, func, or_
 from sqlalchemy.ext.asyncio import async_scoped_session
 from sqlalchemy.future import select
 from sqlalchemy.sql import Select, text
@@ -55,7 +55,13 @@ def __init__(self, session: async_scoped_session) -> None:
         self._session = session
 
     async def add(self, entry: TokenChangeHistoryEntry) -> None:
-        """Record a change to a token."""
+        """Record a change to a token.
+
+        Parameters
+        ----------
+        entry : `gafaelfawr.models.history.TokenChangeHistoryEntry`
+            New entry to add to the database.
+        """
         entry_dict = entry.dict()
 
         # Convert the lists of scopes to the empty string for an empty list
@@ -70,6 +76,19 @@ async def add(self, entry: TokenChangeHistoryEntry) -> None:
         new.event_time = datetime_to_db(entry.event_time)
         self._session.add(new)
 
+    async def delete(self, *, older_than: datetime) -> None:
+        """Delete older entries.
+
+        Parameters
+        ----------
+        older_than : `datetime.datetime`
+            Delete entries created prior to this date.
+        """
+        stmt = delete(TokenChangeHistory).where(
+            TokenChangeHistory.event_time <= datetime_to_db(older_than)
+        )
+        await self._session.execute(stmt)
+
     async def list(
         self,
         *,

tests/cli_test.py

@@ -25,12 +25,15 @@
 
 from gafaelfawr.cli import main
 from gafaelfawr.config import Config, OIDCClient
+from gafaelfawr.constants import CHANGE_HISTORY_RETENTION
 from gafaelfawr.exceptions import InvalidGrantError
 from gafaelfawr.factory import Factory
 from gafaelfawr.models.admin import Admin
+from gafaelfawr.models.history import TokenChange, TokenChangeHistoryEntry
 from gafaelfawr.models.oidc import OIDCAuthorizationCode
 from gafaelfawr.models.token import Token, TokenData, TokenType, TokenUserInfo
 from gafaelfawr.schema import Base
+from gafaelfawr.storage.history import TokenChangeHistoryStore
 from gafaelfawr.storage.token import TokenDatabaseStore
 
 from .support.logging import parse_log
@@ -155,11 +158,34 @@ def test_maintenance(
         created=now - timedelta(minutes=60),
         expires=now - timedelta(minutes=30),
     )
+    new_token_data = TokenData(
+        token=Token(),
+        username="some-user",
+        token_type=TokenType.session,
+        scopes=["read:all", "user:token"],
+        created=now - timedelta(minutes=60),
+        expires=now + timedelta(minutes=30),
+    )
+    old_history_entry = TokenChangeHistoryEntry(
+        token=Token().key,
+        username="other-user",
+        token_type=TokenType.session,
+        scopes=[],
+        expires=now - CHANGE_HISTORY_RETENTION + timedelta(days=10),
+        actor="other-user",
+        action=TokenChange.create,
+        ip_address="127.0.0.1",
+        event_time=now - CHANGE_HISTORY_RETENTION - timedelta(minutes=1),
+    )
 
     async def initialize() -> None:
         async with Factory.standalone(config, engine) as factory:
-            token_store = TokenDatabaseStore(factory.session)
-            await token_store.add(token_data)
+            async with factory.session.begin():
+                token_store = TokenDatabaseStore(factory.session)
+                await token_store.add(token_data)
+                await token_store.add(new_token_data)
+                history_store = TokenChangeHistoryStore(factory.session)
+                await history_store.add(old_history_entry)
 
     event_loop.run_until_complete(initialize())
     runner = CliRunner()
@@ -168,8 +194,13 @@ async def initialize() -> None:
 
     async def check_database() -> None:
         async with Factory.standalone(config, engine) as factory:
-            token_store = TokenDatabaseStore(factory.session)
-            assert await token_store.get_info(token_data.token.key) is None
+            async with factory.session.begin():
+                token_store = TokenDatabaseStore(factory.session)
+                assert await token_store.get_info(token_data.token.key) is None
+                assert await token_store.get_info(new_token_data.token.key)
+                history_store = TokenChangeHistoryStore(factory.session)
+                history = await history_store.list(username="other-user")
+                assert history.entries == []
 
     event_loop.run_until_complete(check_database())
 

tests/services/token_test.py

@@ -10,6 +10,7 @@
 from pydantic import ValidationError
 
 from gafaelfawr.config import Config
+from gafaelfawr.constants import CHANGE_HISTORY_RETENTION
 from gafaelfawr.exceptions import (
     InvalidExpiresError,
     InvalidScopesError,
@@ -26,6 +27,7 @@
     TokenType,
     TokenUserInfo,
 )
+from gafaelfawr.storage.history import TokenChangeHistoryStore
 from gafaelfawr.storage.token import TokenDatabaseStore
 from gafaelfawr.util import current_datetime
 
@@ -1327,7 +1329,7 @@ async def test_invalid_username(factory: Factory) -> None:
 
 
 @pytest.mark.asyncio
-async def test_token_expires(factory: Factory) -> None:
+async def test_expire_tokens(factory: Factory) -> None:
     """Test periodic cleanup of expired tokens."""
     now = datetime.now(tz=timezone.utc)
     session_token_data = TokenData(
@@ -1461,3 +1463,49 @@ async def test_token_expires(factory: Factory) -> None:
             created=unexpired_user_token_data.created,
             expires=unexpired_user_token_data.expires,
         )
+
+
+@pytest.mark.asyncio
+async def test_truncate_history(factory: Factory) -> None:
+    """Test periodic truncation of history."""
+    now = datetime.now(tz=timezone.utc)
+    token_service = factory.create_token_service()
+    session_token_data = await create_session_token(
+        factory, scopes=["admin:token"]
+    )
+    history_store = TokenChangeHistoryStore(factory.session)
+    old_entry = TokenChangeHistoryEntry(
+        token=Token().key,
+        username="other-user",
+        token_type=TokenType.session,
+        scopes=[],
+        expires=now - CHANGE_HISTORY_RETENTION + timedelta(days=10),
+        actor="other-user",
+        action=TokenChange.create,
+        ip_address="127.0.0.1",
+        event_time=now - CHANGE_HISTORY_RETENTION - timedelta(minutes=1),
+    )
+    new_entry = TokenChangeHistoryEntry(
+        token=Token().key,
+        username="other-user",
+        token_type=TokenType.session,
+        scopes=[],
+        expires=now - CHANGE_HISTORY_RETENTION + timedelta(days=5),
+        actor="other-user",
+        action=TokenChange.create,
+        ip_address="127.0.0.1",
+        event_time=now - CHANGE_HISTORY_RETENTION + timedelta(minutes=1),
+    )
+
+    async with factory.session.begin():
+        await history_store.add(old_entry)
+        await history_store.add(new_entry)
+
+    async with factory.session.begin():
+        await token_service.truncate_history()
+
+    async with factory.session.begin():
+        history = await token_service.get_change_history(
+            auth_data=session_token_data, username="other-user"
+        )
+        assert history.entries == [new_entry]